ECS common data mapping (#1274)

dataprovider/enrich.go

@@ -0,0 +1,50 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package dataprovider
+
+import "github.com/elastic/beats/v7/libbeat/beat"
+
+type ElasticCommonDataProvider interface {
+	GetElasticCommonData() (map[string]interface{}, error)
+}
+
+type enricher struct {
+	dataprovider ElasticCommonDataProvider
+}
+
+func NewEnricher(dataprovider ElasticCommonDataProvider) *enricher {
+	return &enricher{
+		dataprovider: dataprovider,
+	}
+}
+
+func (e *enricher) EnrichEvent(event *beat.Event) error {
+	ecsData, err := e.dataprovider.GetElasticCommonData()
+	if err != nil {
+		return err
+	}
+
+	for k, v := range ecsData {
+		_, err := event.PutValue(k, v)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

dataprovider/enrich_test.go

@@ -0,0 +1,97 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package dataprovider
+
+import (
+	"testing"
+
+	"github.com/elastic/beats/v7/libbeat/beat"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEnrichSuccess(t *testing.T) {
+	tests := []struct {
+		name     string
+		data     map[string]interface{}
+		expected map[string]string
+	}{
+		{
+			name: "single value",
+			data: map[string]interface{}{
+				"a-field": "a-value",
+			},
+			expected: map[string]string{
+				"a-field": "a-value",
+			},
+		},
+		{
+			name: "multiple values",
+			data: map[string]interface{}{
+				"some-field":  "some-value",
+				"other-field": "other-value",
+			},
+			expected: map[string]string{
+				"some-field":  "some-value",
+				"other-field": "other-value",
+			},
+		},
+		{
+			name: "internal object",
+			data: map[string]interface{}{
+				"some-field": "some-value",
+				"more-fields": map[string]interface{}{
+					"internal-field": "internal-value",
+				},
+			},
+			expected: map[string]string{
+				"some-field":                 "some-value",
+				"more-fields.internal-field": "internal-value",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			dp := NewMockElasticCommonDataProvider(t)
+			dp.EXPECT().GetElasticCommonData().Return(tt.data, nil)
+
+			ev := &beat.Event{
+				Fields: map[string]interface{}{},
+			}
+			err := NewEnricher(dp).EnrichEvent(ev)
+			assert.NoError(t, err)
+
+			for key, expectedValue := range tt.expected {
+				actualValue, err := ev.GetValue(key)
+				assert.NoError(t, err)
+				assert.Equal(t, expectedValue, actualValue)
+			}
+		})
+	}
+}
+
+func TestEnrichError(t *testing.T) {
+	dp := NewMockElasticCommonDataProvider(t)
+	dp.EXPECT().GetElasticCommonData().Return(nil, assert.AnError)
+
+	ev := &beat.Event{
+		Fields: map[string]interface{}{},
+	}
+	err := NewEnricher(dp).EnrichEvent(ev)
+	assert.Error(t, err)
+}

evaluator/opa_test.go

@@ -45,8 +45,8 @@ func (d *DummyResource) GetMetadata() (fetching.ResourceMetadata, error) {
 func (d *DummyResource) GetData() any {
 	return d
 }
-func (d *DummyResource) GetElasticCommonData() any {
-	return d
+func (d *DummyResource) GetElasticCommonData() (map[string]interface{}, error) {
+	return nil, nil
 }
 
 type OpaTestSuite struct {

resources/fetching/factory/aws_factory.go

@@ -64,8 +64,10 @@ func (w *wrapResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	return mdata, nil
 }
 
-func (w *wrapResource) GetData() any              { return w.wrapped.GetData() }
-func (w *wrapResource) GetElasticCommonData() any { return w.wrapped.GetElasticCommonData() }
+func (w *wrapResource) GetData() any { return w.wrapped.GetData() }
+func (w *wrapResource) GetElasticCommonData() (map[string]interface{}, error) {
+	return w.wrapped.GetElasticCommonData()
+}
 
 func NewCisAwsOrganizationFactory(ctx context.Context, log *logp.Logger, rootCh chan fetching.ResourceInfo, accounts []AwsAccount) FetchersMap {
 	return newCisAwsOrganizationFactory(ctx, log, rootCh, accounts, NewCisAwsFactory)

resources/fetching/factory/aws_factory_test.go

@@ -111,7 +111,9 @@ func subtest(t *testing.T, drain bool) {
 		nameCounts := make(map[string]int)
 		for _, resource := range resources {
 			assert.NotNil(t, resource.GetData())
-			assert.NotNil(t, resource.GetElasticCommonData())
+			cd, err := resource.GetElasticCommonData()
+			assert.NoError(t, err)
+			assert.NotNil(t, cd)
 			mdata, err := resource.GetMetadata()
 			require.NotNil(t, mdata)
 			require.NoError(t, err)
@@ -189,7 +191,7 @@ func mockResource() *fetching.MockResource {
 		AwsAccountId:    "some-id",
 		AwsAccountAlias: "some-alias",
 	}, nil).Once()
-	m.EXPECT().GetElasticCommonData().Return(struct{}{}).Once()
+	m.EXPECT().GetElasticCommonData().Return(map[string]interface{}{}, nil).Once()
 	return &m
 }
 

resources/fetching/fetcher.go

@@ -93,7 +93,7 @@ type CycleMetadata struct {
 type Resource interface {
 	GetMetadata() (ResourceMetadata, error)
 	GetData() any
-	GetElasticCommonData() any
+	GetElasticCommonData() (map[string]interface{}, error)
 }
 
 type ResourceFields struct {
@@ -106,7 +106,6 @@ type ResourceMetadata struct {
 	Type            string `json:"type"`
 	SubType         string `json:"sub_type,omitempty"`
 	Name            string `json:"name,omitempty"`
-	ECSFormat       string `json:"ecsFormat,omitempty"`
 	Region          string `json:"region,omitempty"`
 	AwsAccountId    string `json:"aws_account_id,omitempty"`
 	AwsAccountAlias string `json:"aws_account_alias,omitempty"`

resources/fetching/fetchers/aws/ecr_fetcher.go

@@ -138,4 +138,4 @@ func (res EcrResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	}, nil
 }
 
-func (res EcrResource) GetElasticCommonData() any { return nil }
+func (res EcrResource) GetElasticCommonData() (map[string]interface{}, error) { return nil, nil }

resources/fetching/fetchers/aws/elb_fetcher.go

@@ -119,4 +119,4 @@ func (r ElbResource) GetMetadata() (fetching.ResourceMetadata, error) {
 		Name:    *r.lb.LoadBalancerName,
 	}, nil
 }
-func (r ElbResource) GetElasticCommonData() any { return nil }
+func (r ElbResource) GetElasticCommonData() (map[string]interface{}, error) { return nil, nil }

resources/fetching/fetchers/aws/iam_fetcher.go

@@ -128,4 +128,4 @@ func (r IAMResource) GetMetadata() (fetching.ResourceMetadata, error) {
 		Region:  r.GetRegion(),
 	}, nil
 }
-func (r IAMResource) GetElasticCommonData() any { return nil }
+func (r IAMResource) GetElasticCommonData() (map[string]interface{}, error) { return nil, nil }

resources/fetching/fetchers/aws/kms_fetcher.go

@@ -82,4 +82,4 @@ func (r KmsResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	}, nil
 }
 
-func (r KmsResource) GetElasticCommonData() any { return nil }
+func (r KmsResource) GetElasticCommonData() (map[string]interface{}, error) { return nil, nil }

resources/fetching/fetchers/aws/logging_fetcher.go

@@ -108,7 +108,7 @@ func (r LoggingResource) GetMetadata() (fetching.ResourceMetadata, error) {
 		Region:  r.GetRegion(),
 	}, nil
 }
-func (r LoggingResource) GetElasticCommonData() any { return nil }
+func (r LoggingResource) GetElasticCommonData() (map[string]interface{}, error) { return nil, nil }
 
 func (c ConfigResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	id := fmt.Sprintf("configservice-%s", c.identity.Account)
@@ -124,4 +124,4 @@ func (c ConfigResource) GetData() any {
 	return c.configs
 }
 
-func (c ConfigResource) GetElasticCommonData() any { return nil }
+func (c ConfigResource) GetElasticCommonData() (map[string]interface{}, error) { return nil, nil }

resources/fetching/fetchers/aws/logging_fetcher_test.go

@@ -146,11 +146,13 @@ func TestEnrichedTrailResource_GetMetadata(t *testing.T) {
 	meta, err := r.GetMetadata()
 
 	assert.NoError(t, err)
-	assert.Equal(t, fetching.ResourceMetadata{ID: "test-arn", Type: "cloud-audit", SubType: "aws-trail", Name: "", ECSFormat: ""}, meta)
+	assert.Equal(t, fetching.ResourceMetadata{ID: "test-arn", Type: "cloud-audit", SubType: "aws-trail", Name: ""}, meta)
 	assert.Equal(t, logging.EnrichedTrail{TrailInfo: cloudtrail.TrailInfo{Trail: types.Trail{
 		TrailARN: aws.String("test-arn"),
 	}}}, r.GetData())
-	assert.Equal(t, nil, r.GetElasticCommonData())
+	m, err := r.GetElasticCommonData()
+	assert.NoError(t, err)
+	assert.Empty(t, m)
 }
 
 func TestConfigResource_GetMetadata(t *testing.T) {
@@ -164,7 +166,9 @@ func TestConfigResource_GetMetadata(t *testing.T) {
 	meta, err := r.GetMetadata()
 
 	assert.NoError(t, err)
-	assert.Equal(t, fetching.ResourceMetadata{ID: "configservice-test-account", Type: "cloud-config", SubType: "aws-config", Name: "configservice-test-account", ECSFormat: ""}, meta)
+	assert.Equal(t, fetching.ResourceMetadata{ID: "configservice-test-account", Type: "cloud-config", SubType: "aws-config", Name: "configservice-test-account"}, meta)
 	assert.Nil(t, r.GetData())
-	assert.Equal(t, nil, r.GetElasticCommonData())
+	m, err := r.GetElasticCommonData()
+	assert.NoError(t, err)
+	assert.Empty(t, m)
 }

resources/fetching/fetchers/aws/monitoring_fetcher.go

@@ -103,7 +103,7 @@ func (r MonitoringResource) GetMetadata() (fetching.ResourceMetadata, error) {
 		Region:  awslib.GlobalRegion,
 	}, nil
 }
-func (r MonitoringResource) GetElasticCommonData() any { return nil }
+func (r MonitoringResource) GetElasticCommonData() (map[string]interface{}, error) { return nil, nil }
 
 func (s SecurityHubResource) GetData() any {
 	return s
@@ -119,4 +119,4 @@ func (s SecurityHubResource) GetMetadata() (fetching.ResourceMetadata, error) {
 	}, nil
 }
 
-func (s SecurityHubResource) GetElasticCommonData() any { return nil }
+func (s SecurityHubResource) GetElasticCommonData() (map[string]interface{}, error) { return nil, nil }