feat(system): add missing fields for DEVICE_METADATA

internal/convertor/device/device.go

@@ -52,6 +52,11 @@ type Device struct {
 	CommunityLists  []*routingpolicy.CommunityList
 	RoutePolicies   []*routingpolicy.RoutePolicy
 	AFKEnabled      bool
+
+	// Hwsku and SONiCType are DEVICE_METADATA's hwsku and type, empty when
+	// the device's model or role is not mapped.
+	Hwsku     string
+	SONiCType string
 }
 
 // isAFKenabled checks if the device contains the AFKEnabledTag.
@@ -127,6 +132,16 @@ func NewDevice(dcimInfo *dcim.NetworkDevice, devicesData *repository.AssetsPerDe
 		log.Warn().Msgf("no tacacs found for %s", dcimInfo.Hostname)
 	}
 
+	device.Hwsku, ok = devicesData.SONiCHwsku[dcimInfo.DeviceType.ID]
+	if !ok {
+		log.Warn().Msgf("no SONiC HwSKU mapped to the device type %q of %s", dcimInfo.DeviceType.Model, dcimInfo.Hostname)
+	}
+
+	device.SONiCType, ok = devicesData.SONiCType[dcimInfo.DeviceRole.ID]
+	if !ok {
+		log.Warn().Msgf("no SONiC type mapped to the device role %q of %s", dcimInfo.DeviceRole.Name, dcimInfo.Hostname)
+	}
+
 	return device, nil
 }
 
@@ -147,6 +162,11 @@ func (d *Device) Generateconfigs() error {
 		return fmt.Errorf("convert from Routing Policy to OpenConfig failed: %w", err)
 	}
 
+	aaaConfig, err := tacacsconvertors.TacacsToOpenConfigAAA(d.Tacacs)
+	if err != nil {
+		return fmt.Errorf("convert from TACACS to OpenConfig failed: %w", err)
+	}
+
 	// Assemble global configuration
 	bgpKey := openconfig.NetworkInstance_Protocol_Key{Identifier: openconfig.PolicyTypes_INSTALL_PROTOCOL_TYPE_BGP, Name: "bgp"}
 
@@ -165,24 +185,15 @@ func (d *Device) Generateconfigs() error {
 			},
 		},
 		System: &openconfig.System{
-			Ntp:     ntpconvertors.NTPToOpenconfig(d.NTP),
-			Logging: syslogconvertors.SyslogToOpenconfig(d.Syslog),
+			Hostname: optional(d.Dcim.Hostname),
+			Hwsku:    optional(d.Hwsku),
+			Type:     optional(d.SONiCType),
+			Aaa:      aaaConfig,
+			Ntp:      ntpconvertors.NTPToOpenconfig(d.NTP),
+			Logging:  syslogconvertors.SyslogToOpenconfig(d.Syslog),
 		},
 	}
 
-	if d.Tacacs == nil {
-		log.Warn().Msgf("%s don't have a Tacacs configuration, skip it in OpenconfigConfig", d.Dcim.Hostname)
-	} else {
-		aaa, err := tacacsconvertors.TacacsToOpenConfigAAA(d.Tacacs)
-		if err != nil {
-			return fmt.Errorf("convert from TACACS to OpenConfig failed: %w", err)
-		}
-
-		if aaa != nil {
-			config.System.Aaa = aaa
-		}
-	}
-
 	devJSON, err := ygot.EmitJSON(
 		&config,
 		&ygot.EmitJSONConfig{
@@ -196,6 +207,7 @@ func (d *Device) Generateconfigs() error {
 		return fmt.Errorf("failed to transform an openconfig device specification (%s) into JSON using ygot: %w", d.Dcim.Hostname, err)
 	}
 
+	// IETF configuration
 	d.Config = &GeneratedConfig{
 		Openconfig:     &config,
 		JSONOpenConfig: devJSON,
@@ -230,6 +242,15 @@ func (d *Device) Generateconfigs() error {
 	return nil
 }
 
+// optional returns a pointer to s, or nil when s is empty so that the leaf is
+// left out rather than emitted empty.
+func optional(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return &s
+}
+
 // GetCompactOpenconfigJSON returns OpenConfig result in not indented JSON format.
 // Generated JSON is already indented by Ygot - currently there is no option to not indent the JSON.
 func (d *Device) GetCompactOpenconfigJSON() ([]byte, error) {

internal/convertor/device/device_test.go

@@ -5,14 +5,19 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
+
+	"github.com/criteo/data-aggregation-api/internal/ingestor/repository"
 	"github.com/criteo/data-aggregation-api/internal/model/cmdb/bgp"
 	"github.com/criteo/data-aggregation-api/internal/model/cmdb/ntp"
+	"github.com/criteo/data-aggregation-api/internal/model/cmdb/routingpolicy"
 	"github.com/criteo/data-aggregation-api/internal/model/cmdb/tacacs"
 	"github.com/criteo/data-aggregation-api/internal/model/dcim"
 )
 
 // TestGenerateconfigsSystem checks that the NTP and the TACACS+ configurations,
-// which both live under openconfig-system, are served side by side.
+// which both live under openconfig-system, are served side by side, next to
+// system/config which always holds the hostname.
 func TestGenerateconfigsSystem(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -24,17 +29,17 @@ func TestGenerateconfigsSystem(t *testing.T) {
 			name:     "NTP and TACACS+",
 			ntp:      &ntp.NTP{ServerList: []ntp.Server{{Name: "ntp1", ServerAddress: "10.0.0.1"}}},
 			tacacs:   &tacacs.Tacacs{Passkey: "secret", ServerList: []tacacs.Server{{ServerAddress: "10.10.10.10", Priority: 1, TCPPort: 49}}},
-			wantKeys: []string{"aaa", "ntp"},
+			wantKeys: []string{"aaa", "config", "ntp"},
 		},
 		{
 			name:     "NTP only",
 			ntp:      &ntp.NTP{ServerList: []ntp.Server{{Name: "ntp1", ServerAddress: "10.0.0.1"}}},
-			wantKeys: []string{"ntp"},
+			wantKeys: []string{"config", "ntp"},
 		},
 		{
 			name:     "TACACS+ only",
 			tacacs:   &tacacs.Tacacs{ServerList: []tacacs.Server{{ServerAddress: "10.10.10.10"}}},
-			wantKeys: []string{"aaa"},
+			wantKeys: []string{"aaa", "config"},
 		},
 	}
 
@@ -73,3 +78,157 @@ func TestGenerateconfigsSystem(t *testing.T) {
 		})
 	}
 }
+
+const (
+	deviceTypeID = 10
+	deviceRoleID = 20
+)
+
+func tor() *dcim.NetworkDevice {
+	d := &dcim.NetworkDevice{Hostname: "tor01-01"}
+	d.DeviceType.ID, d.DeviceType.Model = deviceTypeID, "MSN2700"
+	d.DeviceRole.ID, d.DeviceRole.Name = deviceRoleID, "ToR"
+	return d
+}
+
+// assets holds everything NewDevice requires for tor01-01, mappings included.
+func assets() *repository.AssetsPerDevice {
+	return &repository.AssetsPerDevice{
+		BGPsessions:    map[string][]*bgp.Session{"tor01-01": {}},
+		PrefixLists:    map[string][]*routingpolicy.PrefixList{"tor01-01": {}},
+		CommunityLists: map[string][]*routingpolicy.CommunityList{"tor01-01": {}},
+		RoutePolicies:  map[string][]*routingpolicy.RoutePolicy{"tor01-01": {}},
+		SONiCHwsku:     map[int]string{deviceTypeID: "ACS-MSN2700"},
+		SONiCType:      map[int]string{deviceRoleID: "ToRRouter"},
+	}
+}
+
+func TestNewDevice_ResolvesTheSONiCMappings(t *testing.T) {
+	dev, err := NewDevice(tor(), assets())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if dev.Hwsku != "ACS-MSN2700" || dev.SONiCType != "ToRRouter" {
+		t.Errorf("Hwsku, SONiCType = %q, %q", dev.Hwsku, dev.SONiCType)
+	}
+}
+
+// TestNewDevice_UnmappedIsOptional: a device without a mapped model or role
+// is still built, without that DEVICE_METADATA fact.
+func TestNewDevice_UnmappedIsOptional(t *testing.T) {
+	data := assets()
+	data.SONiCHwsku, data.SONiCType = nil, nil
+
+	dev, err := NewDevice(tor(), data)
+	if err != nil {
+		t.Fatalf("NewDevice failed on an unmapped model and role: %v", err)
+	}
+	if dev.Hwsku != "" || dev.SONiCType != "" {
+		t.Errorf("Hwsku, SONiCType = %q, %q, want empty", dev.Hwsku, dev.SONiCType)
+	}
+}
+
+// generateSystem runs Generateconfigs on tor01-01 holding only the system
+// data and returns the emitted JSON.
+func generateSystem(hwsku, sonicType string, ntpConfig *ntp.NTP) (string, error) {
+	d := &Device{
+		mutex:           &sync.Mutex{},
+		Dcim:            &dcim.NetworkDevice{Hostname: "tor01-01"},
+		BGPGlobalConfig: &bgp.BGPGlobal{},
+		NTP:             ntpConfig,
+		Hwsku:           hwsku,
+		SONiCType:       sonicType,
+	}
+	if err := d.Generateconfigs(); err != nil {
+		return "", err
+	}
+	return d.Config.JSONOpenConfig, nil
+}
+
+// systemConfig returns system/config of an emitted device.
+func systemConfig(t *testing.T, raw string) map[string]any {
+	t.Helper()
+	var doc struct {
+		System struct {
+			Config map[string]any `json:"config"`
+		} `json:"system"`
+	}
+	if err := json.Unmarshal([]byte(raw), &doc); err != nil {
+		t.Fatalf("decode %s: %v", raw, err)
+	}
+	return doc.System.Config
+}
+
+// TestGenerateconfigsDeviceMetadata pins the contract with afk-node: the
+// augmented leaves are emitted under system/config with bare names, next to
+// hostname.
+func TestGenerateconfigsDeviceMetadata(t *testing.T) {
+	raw, err := generateSystem("ACS-MSN2700", "ToRRouter", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := map[string]any{"hostname": "tor01-01", "hwsku": "ACS-MSN2700", "type": "ToRRouter"}
+	if diff := cmp.Diff(systemConfig(t, raw), want); diff != "" {
+		t.Errorf("unexpected system/config (-got +want):\n%s\nfull JSON:\n%s", diff, raw)
+	}
+}
+
+// TestGenerateconfigsDeviceMetadata_Unmapped: a device without mappings still
+// gets its hostname, and no empty hwsku or type.
+func TestGenerateconfigsDeviceMetadata_Unmapped(t *testing.T) {
+	raw, err := generateSystem("", "", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := map[string]any{"hostname": "tor01-01"}
+	if diff := cmp.Diff(systemConfig(t, raw), want); diff != "" {
+		t.Errorf("unexpected system/config (-got +want):\n%s", diff)
+	}
+}
+
+// TestGenerateconfigsDeviceMetadata_TypeOutsideThePattern: the YANG pattern is
+// enforced, so a type afk-node does not support never reaches it.
+func TestGenerateconfigsDeviceMetadata_TypeOutsideThePattern(t *testing.T) {
+	for _, sonicType := range []string{"BackEndToRRouter", "not-provisioned"} {
+		if _, err := generateSystem("ACS-MSN2700", sonicType, nil); err == nil {
+			t.Errorf("type %q was emitted, want a validation error", sonicType)
+		}
+	}
+}
+
+// TestGenerateconfigsNTP: the servers land where afk-node reads them,
+// system/ntp/servers/server[].address.
+func TestGenerateconfigsNTP(t *testing.T) {
+	config := &ntp.NTP{ServerList: []ntp.Server{
+		{Name: "ntp1", ServerAddress: "192.0.2.1"},
+		{Name: "ntp2", ServerAddress: "192.0.2.2"},
+	}}
+	raw, err := generateSystem("ACS-MSN2700", "ToRRouter", config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var doc struct {
+		System struct {
+			NTP struct {
+				Servers struct {
+					Server []struct {
+						Address string `json:"address"`
+					} `json:"server"`
+				} `json:"servers"`
+			} `json:"ntp"`
+		} `json:"system"`
+	}
+	if err := json.Unmarshal([]byte(raw), &doc); err != nil {
+		t.Fatal(err)
+	}
+	got := make([]string, 0, len(doc.System.NTP.Servers.Server))
+	for _, s := range doc.System.NTP.Servers.Server {
+		got = append(got, s.Address)
+	}
+	if diff := cmp.Diff(got, []string{"192.0.2.1", "192.0.2.2"}); diff != "" {
+		t.Errorf("unexpected NTP servers (-got +want):\n%s\nfull JSON:\n%s", diff, raw)
+	}
+}

internal/ingestor/cmdb/sonic_mapping.go

@@ -0,0 +1,69 @@
+package cmdb
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/rs/zerolog/log"
+
+	"github.com/criteo/data-aggregation-api/internal/ingestor/netbox"
+	"github.com/criteo/data-aggregation-api/internal/model/cmdb/sonic"
+)
+
+// GetSONiCHwskuMappings returns the DCIM device type to SONiC HwSKU mappings.
+// They are global, so not filtered on the datacenter.
+func GetSONiCHwskuMappings() ([]*sonic.HwskuMapping, error) {
+	response := netbox.NetboxResponse[sonic.HwskuMapping]{}
+
+	err := netbox.Get("/api/plugins/cmdb/sonic-hwsku-mapping/", &response, url.Values{})
+	if err != nil {
+		return nil, fmt.Errorf("SONiC HwSKU mapping fetching failure: %w", err)
+	}
+
+	if len(response.Results) == 0 {
+		log.Warn().Msg("no SONiC HwSKU mapping found")
+	}
+
+	return response.Results, nil
+}
+
+// GetSONiCRoleMappings returns the DCIM device role to SONiC device type
+// mappings. They are global, so not filtered on the datacenter.
+func GetSONiCRoleMappings() ([]*sonic.RoleMapping, error) {
+	response := netbox.NetboxResponse[sonic.RoleMapping]{}
+
+	err := netbox.Get("/api/plugins/cmdb/sonic-role-mapping/", &response, url.Values{})
+	if err != nil {
+		return nil, fmt.Errorf("SONiC role mapping fetching failure: %w", err)
+	}
+
+	if len(response.Results) == 0 {
+		log.Warn().Msg("no SONiC role mapping found")
+	}
+
+	return response.Results, nil
+}
+
+// PrecomputeSONiCHwsku indexes the HwSKUs by DCIM device type ID.
+func PrecomputeSONiCHwsku(mappings []*sonic.HwskuMapping) map[int]string {
+	hwskuPerDeviceType := make(map[int]string, len(mappings))
+	for _, mapping := range mappings {
+		hwskuPerDeviceType[mapping.DeviceType.ID] = mapping.Hwsku
+	}
+
+	return hwskuPerDeviceType
+}
+
+// PrecomputeSONiCType indexes the SONiC device types by DCIM device role ID.
+// A role still "not-provisioned" has no type, and is left out.
+func PrecomputeSONiCType(mappings []*sonic.RoleMapping) map[int]string {
+	typePerRole := make(map[int]string, len(mappings))
+	for _, mapping := range mappings {
+		if mapping.SONiCType == sonic.NotProvisioned {
+			continue
+		}
+		typePerRole[mapping.DeviceRole.ID] = mapping.SONiCType
+	}
+
+	return typePerRole
+}