From 8b470a4dd037ac3dbfa2267b7b31bcae3474a23a Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Tue, 2 Nov 2021 17:32:17 +0100 Subject: [PATCH 1/9] Adds a Telemetry Readmodel for telemetry related collected data --- web/models/host_telemetry.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 web/models/host_telemetry.go diff --git a/web/models/host_telemetry.go b/web/models/host_telemetry.go new file mode 100644 index 000000000..2b82798df --- /dev/null +++ b/web/models/host_telemetry.go @@ -0,0 +1,18 @@ +package models + +import "time" + +type HostTelemetry struct { + AgentID string `gorm:"column:agent_id; primaryKey"` + HostName string `gorm:"column:host_name"` + SLESVersion string `gorm:"column:sles_version"` + CPUCount int `gorm:"column:cpu_count"` + SocketCount int `gorm:"column:socket_count"` + TotalMemoryMB int `gorm:"column:total_memory_mb"` + CloudProvider string `gorm:"column:cloud_provider"` + UpdatedAt time.Time `gorm:"column:updated_at"` +} + +func (HostTelemetry) TableName() string { + return "host_telemetry" +} From dcd3e6e2ffb04fef14ac70027fe4bba71bc0446f Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Tue, 2 Nov 2021 17:33:22 +0100 Subject: [PATCH 2/9] Adds a TelemetryProjector processing different Discoveries --- web/datapipeline/host_telemetry_projector.go | 106 +++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 web/datapipeline/host_telemetry_projector.go diff --git a/web/datapipeline/host_telemetry_projector.go b/web/datapipeline/host_telemetry_projector.go new file mode 100644 index 000000000..7f919f16b --- /dev/null +++ b/web/datapipeline/host_telemetry_projector.go @@ -0,0 +1,106 @@ +package datapipeline + +import ( + "bytes" + "encoding/json" + + log "github.com/sirupsen/logrus" + discoveryModels "github.com/trento-project/trento/agent/discovery/models" + "github.com/trento-project/trento/internal/cloud" + "github.com/trento-project/trento/internal/subscription" + "github.com/trento-project/trento/web/models" + "gorm.io/datatypes" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func NewHostTelemetryProjector(db *gorm.DB) *projector { + telemetryProjector := NewProjector("host_telemetry", db) + + telemetryProjector.AddHandler(HostDiscovery, hostTelemetryProjector_HostDiscoveryHandler) + telemetryProjector.AddHandler(CloudDiscovery, hostTelemetryProjector_CloudDiscoveryHandler) + telemetryProjector.AddHandler(SubscriptionDiscovery, hostTelemetryProjector_SubscriptionDiscoveryHandler) + + return telemetryProjector +} + +func hostTelemetryProjector_HostDiscoveryHandler(dataCollectedEvent *DataCollectedEvent, db *gorm.DB) error { + decoder := payloadDecoder(dataCollectedEvent.Payload) + + var discoveredHost discoveryModels.DiscoveredHost + if err := decoder.Decode(&discoveredHost); err != nil { + log.Errorf("can't decode data: %s", err) + return err + } + + telemetryReadModel := models.HostTelemetry{ + AgentID: dataCollectedEvent.AgentID, + HostName: discoveredHost.HostName, + CPUCount: discoveredHost.CPUCount, + SocketCount: discoveredHost.SocketCount, + TotalMemoryMB: discoveredHost.TotalMemoryMB, + } + + return storeHostTelemetry(db, telemetryReadModel, + "host_name", + "cpu_count", + "socket_count", + "total_memory_mb", + ) +} + +func hostTelemetryProjector_CloudDiscoveryHandler(dataCollectedEvent *DataCollectedEvent, db *gorm.DB) error { + decoder := payloadDecoder(dataCollectedEvent.Payload) + + var discoveredCloud cloud.CloudInstance + if err := decoder.Decode(&discoveredCloud); err != nil { + log.Errorf("can't decode data: %s", err) + return err + } + + telemetryReadModel := models.HostTelemetry{ + AgentID: dataCollectedEvent.AgentID, + CloudProvider: discoveredCloud.Provider, + } + + return storeHostTelemetry(db, telemetryReadModel, "cloud_provider") +} + +func hostTelemetryProjector_SubscriptionDiscoveryHandler(dataCollectedEvent *DataCollectedEvent, db *gorm.DB) error { + decoder := payloadDecoder(dataCollectedEvent.Payload) + + var discoveredSubscription subscription.Subscriptions + + if err := decoder.Decode(&discoveredSubscription); err != nil { + log.Errorf("can't decode data: %s", err) + return err + } + + if len(discoveredSubscription) == 0 { + return nil + } + + telemetryReadModel := models.HostTelemetry{ + AgentID: dataCollectedEvent.AgentID, + SLESVersion: discoveredSubscription[0].Version, + } + + return storeHostTelemetry(db, telemetryReadModel, "sles_version") +} + +func payloadDecoder(payload datatypes.JSON) *json.Decoder { + data, _ := payload.MarshalJSON() + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + + return decoder +} + +func storeHostTelemetry(db *gorm.DB, telemetryReadModel models.HostTelemetry, updateColumns ...string) error { + return db.Clauses(clause.OnConflict{ + Columns: []clause.Column{ + {Name: "agent_id"}, + }, + DoUpdates: clause.AssignmentColumns(updateColumns), + }).Create(&telemetryReadModel).Error +} From b5f0e2b251457dfd842ee8b3366a1bbb6839c1a7 Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Tue, 2 Nov 2021 17:35:06 +0100 Subject: [PATCH 3/9] Adds reference to handled Siscovery types to DataCollectedEvent --- web/datapipeline/data_collected_event.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/web/datapipeline/data_collected_event.go b/web/datapipeline/data_collected_event.go index 6e47356c0..54051369f 100644 --- a/web/datapipeline/data_collected_event.go +++ b/web/datapipeline/data_collected_event.go @@ -9,8 +9,11 @@ import ( ) const ( - ClusterDiscovery = "ha_cluster_discovery" - SAPsystemDiscovery = "sap_system_discovery" + ClusterDiscovery = "ha_cluster_discovery" + SAPsystemDiscovery = "sap_system_discovery" + HostDiscovery = "host_discovery" + SubscriptionDiscovery = "subscription_discovery" + CloudDiscovery = "cloud_discovery" ) type DataCollectedEvent struct { From 1ed5822903c5250cfa44a33450d5e6de86bd70ee Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Tue, 2 Nov 2021 17:36:27 +0100 Subject: [PATCH 4/9] Wired the Telemetry readmodel to the DB and added the new TelemetryProjector to the Registry --- web/app.go | 3 ++- web/datapipeline/projector_registry.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/web/app.go b/web/app.go index 5c00ed94b..11b67c4a3 100644 --- a/web/app.go +++ b/web/app.go @@ -107,7 +107,8 @@ func DefaultDependencies(config *Config) Dependencies { func MigrateDB(db *gorm.DB) error { err := db.AutoMigrate( models.Tag{}, models.SelectedChecks{}, models.ConnectionSettings{}, models.CheckRaw{}, - models.Cluster{}, datapipeline.DataCollectedEvent{}, datapipeline.Subscription{}) + models.Cluster{}, datapipeline.DataCollectedEvent{}, datapipeline.Subscription{}, models.HostTelemetry{}) + if err != nil { return err } diff --git a/web/datapipeline/projector_registry.go b/web/datapipeline/projector_registry.go index 50a043c3b..27ace09cf 100644 --- a/web/datapipeline/projector_registry.go +++ b/web/datapipeline/projector_registry.go @@ -6,8 +6,8 @@ type ProjectorRegistry []Projector // InitInitProjectorsRegistry initialize the ProjectorRegistry func InitProjectorsRegistry(db *gorm.DB) ProjectorRegistry { - return ProjectorRegistry{ NewClustersProjector(db), + NewHostTelemetryProjector(db), } } From 78687c7635f98dd0c06db8d97857a59ff5bf39f8 Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Tue, 2 Nov 2021 17:37:27 +0100 Subject: [PATCH 5/9] Adds a basic init config to set the cwd in tests to the project root --- test/testing.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 test/testing.go diff --git a/test/testing.go b/test/testing.go new file mode 100644 index 000000000..7b9b8940a --- /dev/null +++ b/test/testing.go @@ -0,0 +1,17 @@ +package test + +import ( + "os" + "path" + "runtime" +) + +// importing _ "github.com/trento-project/trento/test" in tests would set the cwd to the root of the repo +func init() { + _, filename, _, _ := runtime.Caller(0) + dir := path.Join(path.Dir(filename), "..") + err := os.Chdir(dir) + if err != nil { + panic(err) + } +} From 3bd3084f777b84dd187dd83ded5d2f7d735f4139 Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Tue, 2 Nov 2021 17:38:22 +0100 Subject: [PATCH 6/9] Adds Tests for TelemetryProjector --- .../host_telemetry_projector_test.go | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 web/datapipeline/host_telemetry_projector_test.go diff --git a/web/datapipeline/host_telemetry_projector_test.go b/web/datapipeline/host_telemetry_projector_test.go new file mode 100644 index 000000000..2b40f48c5 --- /dev/null +++ b/web/datapipeline/host_telemetry_projector_test.go @@ -0,0 +1,131 @@ +package datapipeline + +import ( + "encoding/json" + "testing" + + "github.com/trento-project/trento/agent/discovery/mocks" + + "github.com/stretchr/testify/suite" + _ "github.com/trento-project/trento/test" + "github.com/trento-project/trento/test/helpers" + "github.com/trento-project/trento/web/models" + "gorm.io/gorm" +) + +type HostTelemetryProjectorTestSuite struct { + suite.Suite + db *gorm.DB + tx *gorm.DB +} + +func TestHostTelemetryProjectorTestSuite(t *testing.T) { + suite.Run(t, new(HostTelemetryProjectorTestSuite)) +} + +func (suite *HostTelemetryProjectorTestSuite) SetupSuite() { + suite.db = helpers.SetupTestDatabase(suite.T()) + + suite.db.AutoMigrate(&Subscription{}, &models.HostTelemetry{}) +} + +func (suite *HostTelemetryProjectorTestSuite) TearDownSuite() { + suite.db.Migrator().DropTable(Subscription{}, models.HostTelemetry{}) +} + +func (suite *HostTelemetryProjectorTestSuite) SetupTest() { + suite.tx = suite.db.Begin() +} + +func (suite *HostTelemetryProjectorTestSuite) TearDownTest() { + suite.tx.Rollback() +} + +// Test_HostDiscoveryHandler tests the HostDiscoveryHandler function execution on a HostDiscovery published by an agent +func (s *HostTelemetryProjectorTestSuite) Test_HostDiscoveryHandler() { + discoveredHostMock := mocks.NewDiscoveredHostMock() + + requestBody, _ := json.Marshal(discoveredHostMock) + + hostTelemetryProjector_HostDiscoveryHandler(&DataCollectedEvent{ + ID: 1, + AgentID: "agent_id", + DiscoveryType: HostDiscovery, + Payload: requestBody, + }, s.tx) + + var projectedTelemetry models.HostTelemetry + s.tx.First(&projectedTelemetry) + + s.Equal(discoveredHostMock.HostName, projectedTelemetry.HostName) + s.Equal(discoveredHostMock.CPUCount, projectedTelemetry.CPUCount) + s.Equal(discoveredHostMock.SocketCount, projectedTelemetry.SocketCount) + s.Equal(discoveredHostMock.TotalMemoryMB, projectedTelemetry.TotalMemoryMB) + s.Equal(discoveredHostMock.OSVersion, projectedTelemetry.SLESVersion) + s.Equal("", projectedTelemetry.CloudProvider) +} + +// Test_CloudDiscoveryHandler tests the loudDiscoveryHandler function execution on a CloudDiscovery published by an agent +func (s *HostTelemetryProjectorTestSuite) Test_CloudDiscoveryHandler() { + discoveredCloudMock := mocks.NewDiscoveredCloudMock() + + requestBody, _ := json.Marshal(discoveredCloudMock) + + hostTelemetryProjector_CloudDiscoveryHandler(&DataCollectedEvent{ + ID: 1, + AgentID: "agent_id", + DiscoveryType: CloudDiscovery, + Payload: requestBody, + }, s.tx) + + var projectedTelemetry models.HostTelemetry + s.tx.First(&projectedTelemetry) + + s.Equal("", projectedTelemetry.HostName) + s.Equal(0, projectedTelemetry.CPUCount) + s.Equal(0, projectedTelemetry.SocketCount) + s.Equal(0, projectedTelemetry.TotalMemoryMB) + s.Equal("", projectedTelemetry.SLESVersion) + + expectedCloudProvider := discoveredCloudMock.Provider + s.Equal(expectedCloudProvider, projectedTelemetry.CloudProvider) +} + +// Test_TelemetryProjector tests the TelemetryProjector projects all of the discoveries it is interested in, resulting in a single telemetry readmodel +func (s *HostTelemetryProjectorTestSuite) Test_TelemetryProjector() { + telemetryProjector := NewHostTelemetryProjector(s.tx) + + discoveredCloudMock := mocks.NewDiscoveredCloudMock() + discoveredHostMock := mocks.NewDiscoveredHostMock() + + agentDiscoveries := make(map[string]interface{}) + agentDiscoveries[CloudDiscovery] = discoveredCloudMock + agentDiscoveries[HostDiscovery] = discoveredHostMock + + evtID := int64(1) + + for discoveryType, discoveredData := range agentDiscoveries { + requestBody, _ := json.Marshal(discoveredData) + + telemetryProjector.Project(&DataCollectedEvent{ + ID: evtID, + AgentID: "agent_id", + DiscoveryType: discoveryType, + Payload: requestBody, + }) + evtID++ + } + + var projectedTelemetry models.HostTelemetry + s.tx.First(&projectedTelemetry) + + s.Equal(discoveredHostMock.HostName, projectedTelemetry.HostName) + s.Equal(discoveredHostMock.CPUCount, projectedTelemetry.CPUCount) + s.Equal(discoveredHostMock.SocketCount, projectedTelemetry.SocketCount) + s.Equal(discoveredHostMock.TotalMemoryMB, projectedTelemetry.TotalMemoryMB) + + s.Equal(discoveredHostMock.OSVersion, projectedTelemetry.SLESVersion) + + expectedCloudProvider := discoveredCloudMock.Provider + s.Equal(expectedCloudProvider, projectedTelemetry.CloudProvider) +} From c5ef402b66af6e7061fdf5295b7c1e0f536a7201 Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Tue, 2 Nov 2021 17:39:25 +0100 Subject: [PATCH 7/9] Refactor Tests using paths to filesystem to use the correct cwd --- agent/discovery/collector/client_test.go | 13 +++++++------ agent/discovery/collector/publishing_test.go | 15 ++++++++------- agent/discovery/mocks/discovered_cloud_mock.go | 2 +- agent/discovery/mocks/discovered_cluster_mock.go | 10 +++++----- .../mocks/discovered_subscription_mock.go | 2 +- web/datapipeline/clusters_projector_test.go | 6 +++--- 6 files changed, 25 insertions(+), 23 deletions(-) diff --git a/agent/discovery/collector/client_test.go b/agent/discovery/collector/client_test.go index d3982201c..0ee3cf4da 100644 --- a/agent/discovery/collector/client_test.go +++ b/agent/discovery/collector/client_test.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/afero" "github.com/spf13/viper" "github.com/stretchr/testify/suite" + _ "github.com/trento-project/trento/test" "github.com/trento-project/trento/test/helpers" ) @@ -35,9 +36,9 @@ func (suite *CollectorClientTestSuite) TestCollectorClient_NewClientWithTLS() { EnablemTLS: true, CollectorHost: "localhost", CollectorPort: 8081, - Cert: "../../../test/certs/client-cert.pem", - Key: "../../../test/certs/client-key.pem", - CA: "../../../test/certs/ca-cert.pem", + Cert: "./test/certs/client-cert.pem", + Key: "./test/certs/client-key.pem", + CA: "./test/certs/ca-cert.pem", }) suite.NoError(err) @@ -69,9 +70,9 @@ func (suite *CollectorClientTestSuite) TestCollectorClient_PublishingSuccess() { EnablemTLS: true, CollectorHost: "localhost", CollectorPort: 8081, - Cert: "../../../test/certs/client-cert.pem", - Key: "../../../test/certs/client-key.pem", - CA: "../../../test/certs/ca-cert.pem", + Cert: "./test/certs/client-cert.pem", + Key: "./test/certs/client-key.pem", + CA: "./test/certs/ca-cert.pem", }) suite.NoError(err) diff --git a/agent/discovery/collector/publishing_test.go b/agent/discovery/collector/publishing_test.go index bdc6fdfef..605fb07e6 100644 --- a/agent/discovery/collector/publishing_test.go +++ b/agent/discovery/collector/publishing_test.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/suite" "github.com/trento-project/trento/agent/discovery/mocks" + _ "github.com/trento-project/trento/test" "github.com/trento-project/trento/test/helpers" ) @@ -36,9 +37,9 @@ func (suite *PublishingTestSuite) SetupSuite() { EnablemTLS: true, CollectorHost: "localhost", CollectorPort: 8443, - Cert: "../../../test/certs/client-cert.pem", - Key: "../../../test/certs/client-key.pem", - CA: "../../../test/certs/ca-cert.pem", + Cert: "./test/certs/client-cert.pem", + Key: "./test/certs/client-key.pem", + CA: "./test/certs/ca-cert.pem", }) suite.NoError(err) @@ -52,7 +53,7 @@ func (suite *PublishingTestSuite) TestCollectorClient_PublishingClusterDiscovery discoveredCluster := mocks.NewDiscoveredClusterMock() suite.runDiscoveryScenario(discoveryType, discoveredCluster, func(requestBodyAgainstCollector string) { - suite.assertJsonMatchesJsonFileContent("../../../test/fixtures/discovery/cluster/expected_published_cluster_discovery.json", requestBodyAgainstCollector) + suite.assertJsonMatchesJsonFileContent("./test/fixtures/discovery/cluster/expected_published_cluster_discovery.json", requestBodyAgainstCollector) }) } @@ -61,7 +62,7 @@ func (suite *PublishingTestSuite) TestCollectorClient_PublishingCloudDiscovery() discoveredCloudInstance := mocks.NewDiscoveredCloudMock() suite.runDiscoveryScenario(discoveryType, discoveredCloudInstance, func(requestBodyAgainstCollector string) { - suite.assertJsonMatchesJsonFileContent("../../../test/fixtures/discovery/azure/expected_published_cloud_discovery.json", requestBodyAgainstCollector) + suite.assertJsonMatchesJsonFileContent("./test/fixtures/discovery/azure/expected_published_cloud_discovery.json", requestBodyAgainstCollector) }) } @@ -70,7 +71,7 @@ func (suite *PublishingTestSuite) TestCollectorClient_PublishingHostDiscovery() discoveredHost := mocks.NewDiscoveredHostMock() suite.runDiscoveryScenario(discoveryType, discoveredHost, func(requestBodyAgainstCollector string) { - suite.assertJsonMatchesJsonFileContent("../../../test/fixtures/discovery/host/expected_published_host_discovery.json", requestBodyAgainstCollector) + suite.assertJsonMatchesJsonFileContent("./test/fixtures/discovery/host/expected_published_host_discovery.json", requestBodyAgainstCollector) }) } @@ -79,7 +80,7 @@ func (suite *PublishingTestSuite) TestCollectorClient_PublishingSubscriptionDisc discoveredSubscriptions := mocks.NewDiscoveredSubscriptionsMock() suite.runDiscoveryScenario(discoveryType, discoveredSubscriptions, func(requestBodyAgainstCollector string) { - suite.assertJsonMatchesJsonFileContent("../../../test/fixtures/discovery/subscriptions/expected_published_subscriptions_discovery.json", requestBodyAgainstCollector) + suite.assertJsonMatchesJsonFileContent("./test/fixtures/discovery/subscriptions/expected_published_subscriptions_discovery.json", requestBodyAgainstCollector) }) } diff --git a/agent/discovery/mocks/discovered_cloud_mock.go b/agent/discovery/mocks/discovered_cloud_mock.go index 274a9de56..053e0b092 100644 --- a/agent/discovery/mocks/discovered_cloud_mock.go +++ b/agent/discovery/mocks/discovered_cloud_mock.go @@ -11,7 +11,7 @@ import ( func NewDiscoveredCloudMock() cloud.CloudInstance { metadata := &cloud.AzureMetadata{} - jsonFile, err := os.Open("../../../test/fixtures/discovery/azure/azure_discovery.json") + jsonFile, err := os.Open("./test/fixtures/discovery/azure/azure_discovery.json") if err != nil { panic(err) } diff --git a/agent/discovery/mocks/discovered_cluster_mock.go b/agent/discovery/mocks/discovered_cluster_mock.go index 0884f4343..612e0c7b2 100644 --- a/agent/discovery/mocks/discovered_cluster_mock.go +++ b/agent/discovery/mocks/discovered_cluster_mock.go @@ -4,11 +4,11 @@ import "github.com/trento-project/trento/internal/cluster" func NewDiscoveredClusterMock() cluster.Cluster { cluster, _ := cluster.NewClusterWithDiscoveryTools(&cluster.DiscoveryTools{ - CibAdmPath: "../../../test/fake_cibadmin.sh", - CrmmonAdmPath: "../../../test/fake_crm_mon.sh", - CorosyncKeyPath: "../../../test/authkey", - SBDPath: "../../../test/fake_sbd.sh", - SBDConfigPath: "../../../test/sbd_config", + CibAdmPath: "./test/fake_cibadmin.sh", + CrmmonAdmPath: "./test/fake_crm_mon.sh", + CorosyncKeyPath: "./test/authkey", + SBDPath: "./test/fake_sbd.sh", + SBDConfigPath: "./test/sbd_config", }) return cluster diff --git a/agent/discovery/mocks/discovered_subscription_mock.go b/agent/discovery/mocks/discovered_subscription_mock.go index 62d391a59..6a3f2004b 100644 --- a/agent/discovery/mocks/discovered_subscription_mock.go +++ b/agent/discovery/mocks/discovered_subscription_mock.go @@ -11,7 +11,7 @@ import ( func NewDiscoveredSubscriptionsMock() subscription.Subscriptions { var subs subscription.Subscriptions - jsonFile, err := os.Open("../../../test/fixtures/discovery/subscriptions/subscriptions_discovery.json") + jsonFile, err := os.Open("./test/fixtures/discovery/subscriptions/subscriptions_discovery.json") if err != nil { panic(err) } diff --git a/web/datapipeline/clusters_projector_test.go b/web/datapipeline/clusters_projector_test.go index 79055a85e..a549a2a76 100644 --- a/web/datapipeline/clusters_projector_test.go +++ b/web/datapipeline/clusters_projector_test.go @@ -25,7 +25,7 @@ func TestClustersProjector_ClusterDiscoveryHandler(t *testing.T) { ClusterType: models.ClusterTypeUnknown, }) - jsonFile, err := os.Open("../../test/fixtures/cluster_discovery_hana_scale_up.json") + jsonFile, err := os.Open("./test/fixtures/cluster_discovery_hana_scale_up.json") if err != nil { panic(err) } @@ -54,7 +54,7 @@ func TestClustersProjector_ClusterDiscoveryHandler(t *testing.T) { } func TestTransformClusterListData_HANAScaleUp(t *testing.T) { - jsonFile, err := os.Open("../../test/fixtures/cluster_discovery_hana_scale_up.json") + jsonFile, err := os.Open("./test/fixtures/cluster_discovery_hana_scale_up.json") if err != nil { panic(err) } @@ -76,7 +76,7 @@ func TestTransformClusterListData_HANAScaleUp(t *testing.T) { } func TestTransformClusterListData_Unknown(t *testing.T) { - jsonFile, err := os.Open("../../test/fixtures/cluster_discovery_unknown.json") + jsonFile, err := os.Open("./test/fixtures/cluster_discovery_unknown.json") if err != nil { panic(err) } From dc60f3dde4537f9f26f0e099afc88af5985ae93c Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Wed, 3 Nov 2021 14:04:21 +0100 Subject: [PATCH 8/9] Refactored DiscoveredHost location, moved to internal/hosts --- agent/discovery/host.go | 3 +-- agent/discovery/mocks/discovered_host_mock.go | 8 +++----- .../models => internal/hosts}/discovered_host.go | 2 +- web/datapipeline/host_telemetry_projector.go | 4 ++-- 4 files changed, 7 insertions(+), 10 deletions(-) rename {agent/discovery/models => internal/hosts}/discovered_host.go (95%) diff --git a/agent/discovery/host.go b/agent/discovery/host.go index 1defe225b..d1406e374 100644 --- a/agent/discovery/host.go +++ b/agent/discovery/host.go @@ -7,7 +7,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/trento-project/trento/agent/discovery/collector" - "github.com/trento-project/trento/agent/discovery/models" "github.com/trento-project/trento/internal/consul" "github.com/trento-project/trento/internal/hosts" "github.com/trento-project/trento/version" @@ -50,7 +49,7 @@ func (h HostDiscovery) Discover() (string, error) { var si sysinfo.SysInfo si.GetSysInfo() - host := models.DiscoveredHost{ + host := hosts.DiscoveredHost{ HostIpAddresses: ipAddresses, HostName: h.discovery.host, CPUCount: int(si.CPU.Cpus) * int(si.CPU.Cores), diff --git a/agent/discovery/mocks/discovered_host_mock.go b/agent/discovery/mocks/discovered_host_mock.go index 1338342ec..d73a89f01 100644 --- a/agent/discovery/mocks/discovered_host_mock.go +++ b/agent/discovery/mocks/discovered_host_mock.go @@ -1,11 +1,9 @@ package mocks -import ( - "github.com/trento-project/trento/agent/discovery/models" -) +import "github.com/trento-project/trento/internal/hosts" -func NewDiscoveredHostMock() models.DiscoveredHost { - return models.DiscoveredHost{ +func NewDiscoveredHostMock() hosts.DiscoveredHost { + return hosts.DiscoveredHost{ HostIpAddresses: []string{"10.1.1.4", "10.1.1.5", "10.1.1.6"}, HostName: "thehostnamewherethediscoveryhappened", CPUCount: 64, diff --git a/agent/discovery/models/discovered_host.go b/internal/hosts/discovered_host.go similarity index 95% rename from agent/discovery/models/discovered_host.go rename to internal/hosts/discovered_host.go index 2cbf81671..eef57e214 100644 --- a/agent/discovery/models/discovered_host.go +++ b/internal/hosts/discovered_host.go @@ -1,4 +1,4 @@ -package models +package hosts type DiscoveredHost struct { HostIpAddresses []string `json:"ip_addresses"` diff --git a/web/datapipeline/host_telemetry_projector.go b/web/datapipeline/host_telemetry_projector.go index 7f919f16b..cc0e61c6a 100644 --- a/web/datapipeline/host_telemetry_projector.go +++ b/web/datapipeline/host_telemetry_projector.go @@ -5,8 +5,8 @@ import ( "encoding/json" log "github.com/sirupsen/logrus" - discoveryModels "github.com/trento-project/trento/agent/discovery/models" "github.com/trento-project/trento/internal/cloud" + "github.com/trento-project/trento/internal/hosts" "github.com/trento-project/trento/internal/subscription" "github.com/trento-project/trento/web/models" "gorm.io/datatypes" @@ -27,7 +27,7 @@ func NewHostTelemetryProjector(db *gorm.DB) *projector { func hostTelemetryProjector_HostDiscoveryHandler(dataCollectedEvent *DataCollectedEvent, db *gorm.DB) error { decoder := payloadDecoder(dataCollectedEvent.Payload) - var discoveredHost discoveryModels.DiscoveredHost + var discoveredHost hosts.DiscoveredHost if err := decoder.Decode(&discoveredHost); err != nil { log.Errorf("can't decode data: %s", err) return err From 14e3b0248432c310de044ca131062c0bb2a338ae Mon Sep 17 00:00:00 2001 From: nelsonkopliku Date: Wed, 3 Nov 2021 14:56:15 +0100 Subject: [PATCH 9/9] Refactored HostDiscovery to also publish OS Version --- agent/discovery/host.go | 1 + agent/discovery/mocks/discovered_host_mock.go | 1 + internal/hosts/discovered_host.go | 1 + .../expected_published_host_discovery.json | 1 + web/datapipeline/host_telemetry_projector.go | 26 ++----------------- 5 files changed, 6 insertions(+), 24 deletions(-) diff --git a/agent/discovery/host.go b/agent/discovery/host.go index d1406e374..d9fed9c03 100644 --- a/agent/discovery/host.go +++ b/agent/discovery/host.go @@ -50,6 +50,7 @@ func (h HostDiscovery) Discover() (string, error) { si.GetSysInfo() host := hosts.DiscoveredHost{ + OSVersion: si.OS.Version, HostIpAddresses: ipAddresses, HostName: h.discovery.host, CPUCount: int(si.CPU.Cpus) * int(si.CPU.Cores), diff --git a/agent/discovery/mocks/discovered_host_mock.go b/agent/discovery/mocks/discovered_host_mock.go index d73a89f01..1fabbdbde 100644 --- a/agent/discovery/mocks/discovered_host_mock.go +++ b/agent/discovery/mocks/discovered_host_mock.go @@ -4,6 +4,7 @@ import "github.com/trento-project/trento/internal/hosts" func NewDiscoveredHostMock() hosts.DiscoveredHost { return hosts.DiscoveredHost{ + OSVersion: "15.3", HostIpAddresses: []string{"10.1.1.4", "10.1.1.5", "10.1.1.6"}, HostName: "thehostnamewherethediscoveryhappened", CPUCount: 64, diff --git a/internal/hosts/discovered_host.go b/internal/hosts/discovered_host.go index eef57e214..9894b7cf0 100644 --- a/internal/hosts/discovered_host.go +++ b/internal/hosts/discovered_host.go @@ -1,6 +1,7 @@ package hosts type DiscoveredHost struct { + OSVersion string `json:"os_version"` HostIpAddresses []string `json:"ip_addresses"` HostName string `json:"hostname"` CPUCount int `json:"cpu_count"` diff --git a/test/fixtures/discovery/host/expected_published_host_discovery.json b/test/fixtures/discovery/host/expected_published_host_discovery.json index 1d151cbee..5b212132d 100644 --- a/test/fixtures/discovery/host/expected_published_host_discovery.json +++ b/test/fixtures/discovery/host/expected_published_host_discovery.json @@ -2,6 +2,7 @@ "agent_id": "the-machine-id", "discovery_type": "host_discovery", "payload": { + "os_version": "15.3", "ip_addresses": [ "10.1.1.4", "10.1.1.5", diff --git a/web/datapipeline/host_telemetry_projector.go b/web/datapipeline/host_telemetry_projector.go index cc0e61c6a..4e415e741 100644 --- a/web/datapipeline/host_telemetry_projector.go +++ b/web/datapipeline/host_telemetry_projector.go @@ -7,7 +7,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/trento-project/trento/internal/cloud" "github.com/trento-project/trento/internal/hosts" - "github.com/trento-project/trento/internal/subscription" "github.com/trento-project/trento/web/models" "gorm.io/datatypes" "gorm.io/gorm" @@ -19,7 +18,6 @@ func NewHostTelemetryProjector(db *gorm.DB) *projector { telemetryProjector.AddHandler(HostDiscovery, hostTelemetryProjector_HostDiscoveryHandler) telemetryProjector.AddHandler(CloudDiscovery, hostTelemetryProjector_CloudDiscoveryHandler) - telemetryProjector.AddHandler(SubscriptionDiscovery, hostTelemetryProjector_SubscriptionDiscoveryHandler) return telemetryProjector } @@ -35,6 +33,7 @@ func hostTelemetryProjector_HostDiscoveryHandler(dataCollectedEvent *DataCollect telemetryReadModel := models.HostTelemetry{ AgentID: dataCollectedEvent.AgentID, + SLESVersion: discoveredHost.OSVersion, HostName: discoveredHost.HostName, CPUCount: discoveredHost.CPUCount, SocketCount: discoveredHost.SocketCount, @@ -42,6 +41,7 @@ func hostTelemetryProjector_HostDiscoveryHandler(dataCollectedEvent *DataCollect } return storeHostTelemetry(db, telemetryReadModel, + "sles_version", "host_name", "cpu_count", "socket_count", @@ -66,28 +66,6 @@ func hostTelemetryProjector_CloudDiscoveryHandler(dataCollectedEvent *DataCollec return storeHostTelemetry(db, telemetryReadModel, "cloud_provider") } -func hostTelemetryProjector_SubscriptionDiscoveryHandler(dataCollectedEvent *DataCollectedEvent, db *gorm.DB) error { - decoder := payloadDecoder(dataCollectedEvent.Payload) - - var discoveredSubscription subscription.Subscriptions - - if err := decoder.Decode(&discoveredSubscription); err != nil { - log.Errorf("can't decode data: %s", err) - return err - } - - if len(discoveredSubscription) == 0 { - return nil - } - - telemetryReadModel := models.HostTelemetry{ - AgentID: dataCollectedEvent.AgentID, - SLESVersion: discoveredSubscription[0].Version, - } - - return storeHostTelemetry(db, telemetryReadModel, "sles_version") -} - func payloadDecoder(payload datatypes.JSON) *json.Decoder { data, _ := payload.MarshalJSON() decoder := json.NewDecoder(bytes.NewReader(data))