OB-36691 feat: add secrets manager for increased security on gql token

apps/metricstream/template.yaml

@@ -29,7 +29,7 @@ Parameters:
     Description: >-
       A file hosted in S3 containing list of metrics to stream.
     Default: 's3://observeinc/cloudwatchmetrics/filters/empty.yaml'
-    AllowedPattern: "^s3:\/\/.*"
+    AllowedPattern: "^(s3:\/\/.*)?$"
   OutputFormat:
     Type: String
     Description: >-
@@ -62,14 +62,33 @@ Parameters:
     Description: |
       Buffer incoming data to the specified size, in MiBs, before delivering it
       to the destination.
-
+  ObserveAccountID:
+    Type: String
+    Description: Observe Account Number
+    AllowedPattern: \d*
+  Timestamp:
+    Type: String
+    Description: Timestamp for the metric stream.
+    Default: ''
+    AllowedPattern: "^[0-9]*$"
+  datasourceID:
+    Type: String
+    Description: >-
+      The datastream for this metric stream.
+    AllowedPattern: '\d*'
+  GQLToken:
+    Type: String
+    NoEcho: true
+    Description: >-
+      The token used to retrieve metric configuration.
 Conditions:
   UseStackName: !Equals
     - !Ref NameOverride
     - ''
-  EmptyFilterUri: !Equals
-    - !Ref FilterUri
-    - ""
+  DeployLambda: !Not
+    - !Equals
+      - !Ref datasourceID
+      - ""
 
 Resources:
   DeliveryStreamRole:
@@ -207,6 +226,7 @@ Resources:
       OutputFormat: !Ref OutputFormat
   MetricsRecorderExecutionRole:
     Type: AWS::IAM::Role
+    Condition: DeployLambda
     Properties:
       AssumeRolePolicyDocument:
         Version: '2012-10-17'
@@ -216,7 +236,7 @@ Resources:
               Service: lambda.amazonaws.com
             Action: sts:AssumeRole
       Policies:
-        - PolicyName: S3andMetricStreamPutPolicy
+        - PolicyName: MetricsRecorderPolicy
           PolicyDocument:
             Version: '2012-10-17'
             Statement:
@@ -234,9 +254,13 @@ Resources:
                   - logs:CreateLogStream
                   - logs:PutLogEvents
                 Resource: !GetAtt LambdaLogGroup.Arn
+              - Effect: Allow
+                Action:
+                  - secretsmanager:GetSecretValue
+                Resource: !Ref GQLTokenSecret
   MetricsRecorder:
     Type: AWS::Serverless::Function
-    Condition: EmptyFilterUri
+    Condition: DeployLambda
     Metadata:
       BuildMethod: makefile
     Properties:
@@ -253,21 +277,28 @@ Resources:
       Environment:
         Variables:
           VERBOSITY: 6
-          # fields needed for accessing gql
-          BEARER_TOKEN: "rlA6DnekcdW8OYxyuoaBoRhWXkOec1tC"
-          ACCOUNT_NUMBER: 124375634991
+          ACCOUNT_NUMBER: !Ref ObserveAccountID
+          DATASOURCE_ID: !Ref datasourceID
+          SECRET_NAME: !Ref GQLTokenSecret
           # fields are necessary to update the metric stream
           METRIC_STREAM_NAME: !Ref MetricStream
           FIREHOSE_ARN: !GetAtt DeliveryStream.Arn
           ROLE_ARN: !GetAtt MetricStreamRole.Arn
           OUTPUT_FORMAT: !Ref OutputFormat
+  GQLTokenSecret:
+    Type: AWS::SecretsManager::Secret
+    Condition: DeployLambda
+    Properties:
+      Description: GQL Token Secret
+      Name: !Sub "observe-gql-token-${AWS::StackName}"
+      SecretString: !Ref GQLToken
   StackCreationUpdateCustomResource:
     Type: Custom::StackCreationUpdateTrigger
-    Condition: EmptyFilterUri
+    Condition: DeployLambda
     Properties:
       ServiceToken: !GetAtt MetricsRecorder.Arn
       StackName: !Ref AWS::StackName
-      AccessToken: "rlA6DnekcdW8OYxyuoaBoRhWXkOec1tC"
+      Timestamp: !Ref Timestamp
 Outputs:
   FirehoseArn:
     Description: >-

apps/stack/template.yaml

@@ -110,6 +110,25 @@ Parameters:
       Metrics Stream. If empty, no metrics will be collected.
     Default: 's3://observeinc/cloudwatchmetrics/filters/empty.yaml'
     AllowedPattern: "^(s3:\/\/.*)?$"
+  ObserveAccountID:
+    Type: String
+    Description: Observe Account Number
+    AllowedPattern: \d*
+  Timestamp:
+    Type: String
+    Description: Time when metric stream was created
+    Default: ''
+    AllowedPattern: "^[0-9]*$"
+  datasourceID:
+    Type: String
+    Description: >-
+      The datastream for this metric stream.
+    AllowedPattern: '\d*'
+  GQLToken:
+    Type: String
+    NoEcho: true
+    Description: >-
+      The token used to retrieve metric configuration.
   SourceBucketNames:
     Type: CommaDelimitedList
     Description: >-
@@ -161,10 +180,15 @@ Conditions:
   UseStackName: !Equals
     - !Ref NameOverride
     - ""
-  EnableMetricStream: !Not
-    - !Equals
-      - !Ref MetricStreamFilterUri
-      - ""
+  EnableMetricStream: !Or
+    - !Not
+      - !Equals
+        - !Ref MetricStreamFilterUri
+        - ""
+    - !Not
+      - !Equals
+        - !Ref datasourceID
+        - ""
 
 Resources:
   Topic:
@@ -311,6 +335,10 @@ Resources:
       Parameters:
         BucketArn: !GetAtt Bucket.Arn
         FilterUri: !Ref MetricStreamFilterUri
+        ObserveAccountID: !Ref ObserveAccountID
+        Timestamp: !Ref Timestamp
+        GQLToken: !Ref GQLToken
+        datasourceID: !Ref datasourceID
         NameOverride: !If
           - UseStackName
           - !Sub "${AWS::StackName}-MetricStream"

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.41.2
 	github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.37.3
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.4
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3
 	github.com/aws/smithy-go v1.21.0
 	github.com/go-logr/logr v1.4.2

go.sum

@@ -38,6 +38,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 h1:246A4lSTXWJw/
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15/go.mod h1:haVfg3761/WF7YPuJOER2MP0k4UAXyHaLclKXB6usDg=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 h1:hT8ZAZRIfqBqHbzKTII+CIiY8G2oC9OpLedkZ51DWl8=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.4 h1:EoPbZg+DGTRqKKhwk5uDviV9yvx65r1kyoNNC02ZH4Y=
+github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.33.4/go.mod h1:WyLS5qwXHtjKAONYZq/4ewdd+hcVsa3LBu77Ow5uj3k=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 h1:Vjqy5BZCOIsn4Pj8xzyqgGmsSqzz7y/WXbN3RgOoVrc=
 github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3/go.mod h1:L0enV3GCRd5iG9B64W35C4/hwsCB00Ib+DKVGTadKHI=
 github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 h1:BXx0ZIxvrJdSgSvKTZ+yRBeSqqgPM89VPlulEcl37tM=

pkg/handler/metricsrecorder/handler.go

@@ -16,6 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
 )
 
 type GraphQLRequest struct {
@@ -25,17 +26,19 @@ type GraphQLRequest struct {
 type Handler struct {
 	Logger           logr.Logger
 	metricStreamName string
-	BearerToken      string
 	FirehoseArn      string
 	RoleArn          string
 	OutputFormat     string
-	AccountNumber    int
+	AccountNumber    string
+	datasourceID     string
+	SecretName       string
 }
 
 type Config struct {
 	MetricStreamName string `env:"METRIC_STREAM_NAME,required"`
-	BearerToken      string `env:"BEARER_TOKEN,required"`
-	AccountNumber    int    `env:"ACCOUNT_NUMBER,required"`
+	AccountNumber    string `env:"ACCOUNT_NUMBER,required"`
+	datasourceID     string `env:"DATASOURCE_ID,required"`
+	SecretName       string `env:"SECRET_NAME,required"`
 	FirehoseArn      string `env:"FIREHOSE_ARN,required"`
 	RoleArn          string `env:"ROLE_ARN,required"`
 	OutputFormat     string `env:"OUTPUT_FORMAT,required"`
@@ -49,6 +52,10 @@ type MetricsListItem struct {
 
 func convertToMetricStreamFilters(MetricsList []MetricsListItem) []types.MetricStreamFilter {
 
+	if len(MetricsList) == 0 {
+		return []types.MetricStreamFilter{}
+	}
+
 	var metricsFilters []types.MetricStreamFilter
 
 	for _, service := range MetricsList {
@@ -80,8 +87,9 @@ func New(cfg *Config, logger logr.Logger) (Handler, error) {
 	return Handler{
 		Logger:           logger,
 		metricStreamName: cfg.MetricStreamName,
-		BearerToken:      cfg.BearerToken,
 		AccountNumber:    cfg.AccountNumber,
+		SecretName:       cfg.SecretName,
+		datasourceID:     cfg.datasourceID,
 		FirehoseArn:      cfg.FirehoseArn,
 		RoleArn:          cfg.RoleArn,
 		OutputFormat:     cfg.OutputFormat,
@@ -102,10 +110,6 @@ func (h Handler) Invoke(ctx context.Context, payload []byte) ([]byte, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error unmarshalling JSON: %v", err)
 	}
-
-	logger.V(4).Info("handling request")
-	logger.Info("Parsed JSON", "data", rawResult)
-
 	var req Request
 
 	// we cannot return errors until we have parsed out the json for the request,
@@ -123,20 +127,35 @@ func (h Handler) Invoke(ctx context.Context, payload []byte) ([]byte, error) {
 		return []byte{}, nil
 	}
 
-	metricStreamName := h.metricStreamName
+	cfg, err := config.LoadDefaultConfig(ctx)
+	if err != nil {
+		h.reportStatus(req, false, "failed to load AWS config")
+		return nil, fmt.Errorf("failed to load AWS config: %w", err)
+	}
+
+	secretName := h.SecretName
+
+	svc := secretsmanager.NewFromConfig(cfg)
+	secretValue, err := svc.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{
+		SecretId: &secretName,
+	})
+	if err != nil {
+		h.reportStatus(req, false, fmt.Sprintf("failed to retrieve secret value %s", err))
+		return nil, fmt.Errorf("failed to get secret value: %w", err)
+	}
+
+	token := *secretValue.SecretString
 
-	// get necessary info for api call (tbd what this is exactly, acct no + gql token)
-	// call observe api to get metrics settings from postgres
-	token := h.BearerToken
+	metricStreamName := h.metricStreamName
 	accountNumber := h.AccountNumber
-	logger.V(4).Info("calling observe api", "token", token)
+	datasourceID := h.datasourceID
 	logger.V(4).Info("calling observe api", "accountNumber", accountNumber)
 
-	fullToken := fmt.Sprintf("Bearer %d %s", accountNumber, token)
+	fullToken := fmt.Sprintf("Bearer %s %s", accountNumber, token)
 
-	query := `
+	query := fmt.Sprintf(`
 		{
-			datasource(id: "41752136") {
+			datasource(id: "%s") {
 				name
 				variables {
 					details {
@@ -150,15 +169,18 @@ func (h Handler) Invoke(ctx context.Context, payload []byte) ([]byte, error) {
 				}
 			}
 		}
-	`
+	`, datasourceID)
+
+	// print out the query for debugging purposes
+	logger.V(4).Info("calling observe api", "query", query)
 
 	gqlRequest := GraphQLRequest{Query: query}
 	jsonData, err := json.Marshal(gqlRequest)
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal request into gql: %w", err)
 	}
 
-	host := fmt.Sprintf("%d.observe-eng.com", accountNumber)
+	host := fmt.Sprintf("%s.observe-eng.com", accountNumber)
 	url := fmt.Sprintf("https://%s/v1/meta", host)
 
 	request, reqErr := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
@@ -187,6 +209,8 @@ func (h Handler) Invoke(ctx context.Context, payload []byte) ([]byte, error) {
 		return nil, fmt.Errorf("error unmarshalling JSON from GQL response: %w", err)
 	}
 
+	logger.V(4).Info("response from observe api", "result", result)
+
 	var metricSelection []MetricsListItem
 
 	if len(result.Data.Datasource.Variables) > 0 {
@@ -195,12 +219,6 @@ func (h Handler) Invoke(ctx context.Context, payload []byte) ([]byte, error) {
 
 	MetricsFilters := convertToMetricStreamFilters(metricSelection)
 
-	cfg, err := config.LoadDefaultConfig(ctx)
-	if err != nil {
-		h.reportStatus(req, false, "failed to load AWS config")
-		return nil, fmt.Errorf("failed to load AWS config: %w", err)
-	}
-
 	// Create a cloudwatch client
 	cwClient := cloudwatch.NewFromConfig(cfg)
 
@@ -217,8 +235,8 @@ func (h Handler) Invoke(ctx context.Context, payload []byte) ([]byte, error) {
 		return nil, fmt.Errorf("failed to add filters to metric stream: %w", err)
 	}
 
-	logger.V(4).Info("successfully wrote metrics to s3")
-	err = h.reportStatus(req, true, "successfully wrote metrics to s3")
+	logger.V(4).Info("successfully wrote metrics to metric stream")
+	err = h.reportStatus(req, true, "successfully wrote metrics to metric stream")
 	if err != nil {
 		return nil, fmt.Errorf("failed to report status to cloudformation: %w", err)
 	}