[NR-347440] CFN template validate lambda integration logs

.github/workflows/run-e2e-tests.yaml

@@ -3,16 +3,55 @@ name: E2E Test Workflow
 on:
   pull_request:
     branches:
-      - develop
       - main
+      - develop
 
 jobs:
-  run-e2e-tests:
+  run-e2e-tests-cloudwatch:
     runs-on: ubuntu-latest
     permissions:
       id-token: write
       contents: write
+    strategy:
+      matrix:
+        test-case: [test_logs_for_filter_pattern, test_logs_for_secret_manager, test_logs_for_invalid_log_group]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.22'
+
+      - name: Install AWS SAM CLI
+        run: |
+          pip install aws-sam-cli
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_E2E_ROLE }}
+          aws-region: us-east-1
 
+      - name: Run e2e tests
+        env:
+          NEW_RELIC_USER_KEY: ${{ secrets.NEW_RELIC_USER_KEY }}
+          NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+        run: |
+          cd e2e-tests/common-scripts
+          ./build-templates.sh
+          cd ..
+          ./lambda-cloudwatch-trigger.sh ${{ matrix.test-case }}
+
+  run-e2e-tests-s3:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    strategy:
+      matrix:
+        test-case: [test_logs_for_prefix, test_logs_for_secret_manager, test_logs_for_invalid_bucket_name]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -36,22 +75,28 @@ jobs:
         env:
           NEW_RELIC_USER_KEY: ${{ secrets.NEW_RELIC_USER_KEY }}
           NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
-          S3_BUCKET: unified-lambda-e2e-test-templates
         run: |
-          cd e2e-tests/
+          cd e2e-tests/common-scripts
           ./build-templates.sh
-
-          echo "Running s3 and cloudwatch trigger tests parallely"
-          ./lambda-cloudwatch-trigger.sh &
-          pid1=$!
-          echo "Testing setting up cloudwatch trigger with PID: $pid1"
-
-          ./lambda-s3-trigger.sh &
-          pid2=$!
-          echo "Testing setting up s3 trigger with PID: $pid2"
-
-          wait $pid1
-          wait $pid2
+          cd ..
+          ./lambda-s3-trigger.sh ${{ matrix.test-case }}
+
+  clean-up:
+    needs: [run-e2e-tests-cloudwatch, run-e2e-tests-s3]
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - name: Install AWS SAM CLI
+        run: |
+          pip install aws-sam-cli
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ secrets.AWS_E2E_ROLE }}
+          aws-region: us-east-1
 
       - name: Delete Resources
         env:

e2e-tests/build-templates.sh → e2e-tests/common-scripts/build-templates.sh

@@ -11,7 +11,7 @@ for TEMPLATE_FILE in "${TEMPLATES[@]}"; do
   BASE_NAME=$(basename "$TEMPLATE_FILE" .yaml)
   BUILD_DIR="$BUILD_DIR_BASE/$BASE_NAME"
 
-  sam build -u --template-file "../$TEMPLATE_FILE" --build-dir "$BUILD_DIR"
+  sam build -u --template-file "../../$TEMPLATE_FILE" --build-dir "$BUILD_DIR"
   sam package --s3-bucket "$S3_BUCKET" --template-file "$BUILD_DIR/template.yaml" --output-template-file "$BUILD_DIR/$TEMPLATE_FILE"
 
 done

e2e-tests/common-scripts/config-file.cfg

@@ -0,0 +1,48 @@
+# test resources
+NEW_RELIC_REGION=US
+NEW_RELIC_ACCOUNT_ID=2813435
+S3_BUCKET=aws-unified-lambda-e2e-test-templates
+
+# test case constants
+CLOUDWATCH_FILTER_PATTERN_CASE=e2e-cloudwatch-filter-pattern-test
+CLOUDWATCH_SECRET_MANAGER_CASE=e2e-cloudwatch-secret-manager-test
+CLOUDWATCH_INVALID_LOG_GROUP_CASE=e2e-cloudwatch-invalid-log-group-test
+S3_PREFIX_CASE=e2e-s3-prefix-test
+S3_SECRET_MANAGER_CASE=e2e-s3-secret-manager-test
+S3_INVALID_BUCKET_CASE=e2e-s3-invalid-bucket-test
+
+# build constants
+BUILD_DIR_BASE=.aws-sam/build
+
+# template constants
+LAMBDA_TEMPLATE=lambda-template.yaml
+LAMBDA_TEMPLATE_BUILD_DIR=common-scripts/.aws-sam/build/lambda-template
+LAMBDA_LOGICAL_RESOURCE_ID=NewRelicLogsServerlessLogForwarder
+
+# deployment constants for s3 bucket cases
+S3_BUCKET_NAME=aws-unified-lambda-e2e-test-bucket
+S3_BUCKET_NAME_SECRET_MANAGER=aws-unified-lambda-e2e-test-bucket-secret-manager-case
+S3_BUCKET_NAME_INVALID=aws-unified-lambda-e2e-test-bucket-non-existent
+S3_BUCKET_PREFIX=LogFolder
+S3_BUCKET_PREFIX_INVALID=InvalidFolder
+S3_BUCKET_OBJECT_NAME=TestTextFile.txt
+S3_BUCKET_OBJECT_NAME_FOR_INVALID_CASE=TestTextFileInvalid.txt
+
+# deployment constants for cloudwatch log group cases
+LOG_GROUP_NAME=aws-unified-lambda-e2e-test-log-group
+LOG_GROUP_NAME_SECRET_MANAGER=aws-unified-lambda-e2e-test-log-group-secret-manager
+LOG_GROUP_NAME_INVALID=aws-unified-lambda-e2e-test-log-group-invalid
+LOG_GROUP_FILTER_PATTERN=ERROR
+LOG_STREAM_NAME=aws-unified-lambda-e2e-test-stream
+
+# fetch nr logs constants
+LOG_MESSAGE_CLOUDWATCH="This is cloudwatch logs"
+LOG_MESSAGE_S3="This is s3 logs"
+ATTRIBUTE_KEY_CLOUDWATCH=logStream
+ATTRIBUTE_KEY_S3=logObjectKey
+TIME_RANGE="5 minutes"
+MAX_RETRIES=5
+SLEEP_TIME=30
+MAX_SLEEP_TIME=180
+COMMON_ATTRIBUTE_KEY=testKey
+COMMON_ATTRIBUTE_VALUE=testValue

e2e-tests/common-scripts/logs-scripts.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+source config-file.cfg
+source stack-scripts.sh
+
+validate_logs_in_new_relic() {
+  user_key=$1
+  account_id=$2
+  attribute_key=$3
+  attribute_value=$4
+  log_message=$5
+
+  sleep_time=$SLEEP_TIME
+  attempt=1
+
+  while [[ $attempt -lt $MAX_RETRIES ]]; do
+    echo "Fetching logs from new relic for $attribute_key: $attribute_value"
+    sleep "$sleep_time"
+    response=$(fetch_new_relic_logs_api "$user_key" "$account_id" "$attribute_key" "$attribute_value")
+
+    if echo "$response" | grep -q "$log_message"; then
+      echo "Log event successfully found in New Relic."
+      validate_logs_meta_data "$response"
+      return 0
+    fi
+
+    if (( sleep_time < MAX_SLEEP_TIME )); then
+      sleep_time=$(( sleep_time * 2 ))
+    fi
+    echo "Log event not found in New Relic. Retrying in $sleep_time seconds..."
+    attempt=$((attempt + 1))
+  done
+
+  exit_with_error "Log event with $attribute_key: $attribute_value not found in New Relic. Error Received: $response"
+}
+
+validate_logs_not_present() {
+  user_key=$1
+  account_id=$2
+  attribute_key=$3
+  attribute_value=$4
+  log_message=$5
+
+  max_retries=3
+  sleep_time=30
+  attempt=1
+
+  while [[ $attempt -lt $max_retries ]]; do
+    echo "Fetching logs from new relic for $attribute_key: $attribute_value"
+    sleep "$sleep_time"
+    response=$(fetch_new_relic_logs_api "$user_key" "$account_id" "$attribute_key" "$attribute_value")
+
+    if echo "$response" | grep -q "$log_message"; then
+      exit_with_error "Log event found in New Relic. Validation failed"
+    fi
+
+    echo "Log event not found in New Relic. Retrying in $sleep_time seconds..."
+    attempt=$((attempt + 1))
+  done
+
+  echo "Log event with $attribute_key: $attribute_value not found in New Relic. Validation succeeded"
+}
+
+fetch_new_relic_logs_api() {
+  user_key=$1
+  account_id=$2
+  attribute_key=$3
+  attribute_value=$4
+
+  nrql_query="SELECT * FROM Log WHERE $attribute_key LIKE '%$attribute_value%' SINCE $TIME_RANGE ago"
+  query='{"query":"query($id: Int!, $nrql: Nrql!) { actor { account(id: $id) { nrql(query: $nrql) { results } } } }","variables":{"id":'$account_id',"nrql":"'$nrql_query'"}}'
+
+  response=$(curl -s -X POST \
+    -H "Content-Type: application/json" \
+    -H "API-Key: $user_key" \
+    -d "$query" \
+    https://api.newrelic.com/graphql)
+
+  echo "$response"
+}
+
+create_log_message() {
+  log_message=$1
+  filter_pattern=$2
+
+  UUID=$(uuidgen)
+  echo "RequestId: $UUID, message: $log_message, filter: $filter_pattern"
+}
+
+validate_logs_meta_data (){
+  response=$1
+
+  # Validate common attributes
+  if ! echo "$response" | grep -q "\"$COMMON_ATTRIBUTE_KEY\":\"$COMMON_ATTRIBUTE_VALUE\""; then
+    exit_with_error "Common attribute $COMMON_ATTRIBUTE_KEY with value $COMMON_ATTRIBUTE_VALUE not found in New Relic logs."
+  fi
+  echo "Common attributes validated successfully."
+
+  # toDo: add entity synthesis validation post confirmation on this
+}