Testing - Update Weathertop with new data ingestion pipeline (#5384)

* updated config * Error * Error * Adding new data stack for streaming log data cross-account * CDK updates * Data stack fully operational * fully working data streams * Adding log processing feature in Consumer Stack * Both stacks fully operational * working versions of everything * Reformatting with black * isort * isort * removing hardcoded ARN's * updates for weathertop working, tweaks * updated diagrams * Delete test/sqs_lambda_to_batch_fargate/test.txt * changes for David * changes for David * changes for David * changes for David * updates * updates * status into variable * status into variable * changes * changes * changes * updates to text
awsdocs · Dec 15, 2023 · 6cbff8b · 6cbff8b
1 parent 68d9530
commit 6cbff8b
Show file tree

Hide file tree

Showing 27 changed files with 489 additions and 387 deletions.
diff --git a/test/architecture_diagrams/pdf/weathertop-comp-1.pdf b/test/architecture_diagrams/pdf/weathertop-comp-1.pdf
diff --git a/test/architecture_diagrams/pdf/weathertop-comp-2.pdf b/test/architecture_diagrams/pdf/weathertop-comp-2.pdf
diff --git a/test/architecture_diagrams/pdf/weathertop-high-level-architecture.pdf b/test/architecture_diagrams/pdf/weathertop-high-level-architecture.pdf
diff --git a/test/architecture_diagrams/png/weathertop-comp-2.png b/test/architecture_diagrams/png/weathertop-comp-2.png
diff --git a/test/architecture_diagrams/png/weathertop-high-level-architecture.png b/test/architecture_diagrams/png/weathertop-high-level-architecture.png
diff --git a/test/config/README.md b/test/config/README.md
@@ -0,0 +1,12 @@
+# Config
+This directory contains the configuration for this stack.
+
+### [resources.yaml](resources.yaml)
+Names of AWS resources in this testing stack with cross-account configuration.
+* `topic_name` [String] Where scheduled testing events are published. Allows cross-account `Publish`.
+* `bucket_name` [String] Where testing results are published. Allows cross-account `PutObject`.
+
+### [targets.yaml](targets.yaml).
+Name and configuration for specific SDK languages. Includes:
+* `account_id` [String] The AWS Account where a set of AWS SDK test are running
+* `status` [String] Whether the AWS SDK tests are running
diff --git a/test/config/resources.yaml b/test/config/resources.yaml
@@ -0,0 +1,2 @@
+topic_name: "aws-weathertop-central-sns-fanout-topic"
+bucket_name: "aws-weathertop-central-log-bucket"
diff --git a/test/config/targets.yaml b/test/config/targets.yaml
@@ -0,0 +1,36 @@
+cpp:
+  account_id: "770244195820"
+  status: "disabled"
+dotnetv3:
+  account_id: "441997275833"
+  status: "disabled"
+gov2:
+  account_id: "234521034040"
+  status: "disabled"
+javascriptv3:
+  account_id: "875008041426"
+  status: "disabled"
+javav2:
+  account_id: "814548047983" # back-up "667348412466"
+  status: "disabled"
+kotlin:
+  account_id: "814548047983" # back-up "667348412466"
+  status: "disabled"
+php:
+  account_id: "733931915187"
+  status: "disabled"
+python:
+  account_id: "664857444588"
+  status: "disabled"
+ruby:
+  account_id: "616362385685"
+  status: "disabled"
+rust_dev_preview:
+  account_id: "050288538048"
+  status: "disabled"
+sap-abap:
+  account_id: "099736152523"
+  status: "disabled"
+swift:
+  account_id: "637397754108"
+  status: "disabled"
diff --git a/test/eventbridge_rule_with_sns_fanout/README.md b/test/eventbridge_rule_with_sns_fanout/README.md
@@ -18,13 +18,10 @@ Specifically, it deploys a scheduled Amazon EventBridge rule that publishes a me
 * AWS access key and secret for AWS user with permissions to create the preceding resources
 * Successfully written [system parameters](#storing-system-parameters)
 
-### Storing system parameters
+### Updating configuration data
 
-Before you get started, execute [store_system_parameters.py](store_system_params.py) as described in the code comments:
+Before you get started, update [config/resources.yaml](../config/resources.yaml) and [config/targets.yaml](../config/targets.yaml) to include logical names representing test targets and their corresponding AWS Account ID and enabled status. 
 
-```
-python3 store_system_parameters.py
-```
 ---
 ## AWS CDK setup and deployment
 

diff --git a/test/eventbridge_rule_with_sns_fanout/app.py b/test/eventbridge_rule_with_sns_fanout/app.py
@@ -1,15 +1,18 @@
 #!/usr/bin/env python3
 
-from aws_cdk import App
-import aws_cdk as cdk
+import os
 
+import aws_cdk as cdk
+from aws_cdk import App
 from producer_stack.producer_stack import ProducerStack
 
-import os
-
 app = App()
-ProducerStack(app, "ProducerStack",
-    env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')),
-    )
+ProducerStack(
+    app,
+    "ProducerStack",
+    env=cdk.Environment(
+        account=os.getenv("CDK_DEFAULT_ACCOUNT"), region=os.getenv("CDK_DEFAULT_REGION")
+    ),
+)
 
-app.synth()
+app.synth()
diff --git a/test/eventbridge_rule_with_sns_fanout/producer_stack/__init__.py b/test/eventbridge_rule_with_sns_fanout/producer_stack/__init__.py
diff --git a/test/eventbridge_rule_with_sns_fanout/producer_stack/producer_stack.py b/test/eventbridge_rule_with_sns_fanout/producer_stack/producer_stack.py
@@ -1,96 +1,115 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: MIT-0
 
-import boto3
-from aws_cdk import (
-    aws_iam as iam,
-    aws_events as events,
-    aws_events_targets as targets,
-    aws_sns as sns,
-    aws_kinesis as kinesis,
-    aws_sns_subscriptions as subscriptions,
-    aws_logs as logs,
-    Aws,
-    Stack
-)
+import yaml
+from aws_cdk import Aws, CfnOutput, Duration, Size, Stack
+from aws_cdk import aws_events as events
+from aws_cdk import aws_events_targets as targets
+from aws_cdk import aws_iam as iam
+from aws_cdk import aws_s3 as s3
+from aws_cdk import aws_sns as sns
 from constructs import Construct
 
+
 class ProducerStack(Stack):
     def __init__(self, scope: Construct, id: str, **kwargs) -> None:
         super().__init__(scope, id, **kwargs)
+        acct_config = self.get_yaml_config("../../config/targets.yaml")
+        resource_config = self.get_yaml_config("../../config/resources.yaml")
+        topic_name = resource_config["topic_name"]
+        bucket_name = resource_config["bucket_name"]
+        topic = self.init_get_topic(topic_name)
+        self.sns_permissions(topic)
+        self.init_subscribe_permissions(topic, acct_config)
+        self.init_publish_permissions(topic, acct_config)
+        bucket = self.init_create_bucket(bucket_name)
+        self.init_cross_account_log_role(acct_config, bucket)
 
-        client = boto3.client('ssm')
-
-        onboarded_languages = [
-            'ruby',
-            'javav2',
-            # 'javascriptv3'
-            'gov2',
-            'python',
-            'dotnetv3',
-            # 'kotlin'
-            # 'rust_dev_preview'
-            # 'swift'
-            # 'cpp'
-            # 'gov2'
-            # 'sap-abap'
-        ]
-
-        account_ids = []
-        for language_name in onboarded_languages:
-            response = client.get_parameter(Name=f'/account-mappings/{language_name}', WithDecryption=True)
-            account_ids.append(response['Parameter']['Value'])
+    def get_yaml_config(self, filepath):
+        with open(filepath, "r") as file:
+            data = yaml.safe_load(file)
+        return data
 
-        # Create a new Amazon Simple Notification Service (Amazon SNS) topic.
-        topic = sns.Topic(self, "fanout-topic")
+    def init_get_topic(self, topic_name):
+        topic = sns.Topic(self, "fanout-topic", topic_name=topic_name)
+        return topic
 
-        # Create a new Amazon EventBridge rule.
+    def init_rule(self, topic):
         rule = events.Rule(
             self,
             "trigger-rule",
             schedule=events.Schedule.cron(
-                # Uncomment after testing.
-                # minute="0",
-                # hour="22",
-                # week_day="FRI",
+                minute="0",
+                hour="22",
+                week_day="FRI",
             ),
         )
-
-        # Add a target to the EventBridge rule to publish a message to the SNS topic.
         rule.add_target(targets.SnsTopic(topic))
 
+    def sns_permissions(self, topic):
         # Set up base Amazon SNS permissions.
         sns_permissions = iam.PolicyStatement()
         sns_permissions.add_any_principal()
         sns_permissions.add_actions(
-                        "SNS:Publish",
-                        "SNS:RemovePermission",
-                        "SNS:SetTopicAttributes",
-                        "SNS:DeleteTopic",
-                        "SNS:ListSubscriptionsByTopic",
-                        "SNS:GetTopicAttributes",
-                        "SNS:AddPermission",
-                        "SNS:Subscribe"
-                    )
+            "SNS:AddPermission",
+            "SNS:DeleteTopic",
+            "SNS:GetTopicAttributes",
+            "SNS:ListSubscriptionsByTopic",
+            "SNS:SetTopicAttributes",
+            "SNS:Subscribe",
+            "SNS:RemovePermission",
+            "SNS:Publish",
+        )
         sns_permissions.add_resources(topic.topic_arn)
-        sns_permissions.add_condition("StringEquals", {"AWS:SourceOwner": Aws.ACCOUNT_ID})
+        sns_permissions.add_condition(
+            "StringEquals", {"AWS:SourceOwner": Aws.ACCOUNT_ID}
+        )
         topic.add_to_resource_policy(sns_permissions)
 
-        # Set up cross-account Subscription permissions for every onboarded language.
+    def init_subscribe_permissions(self, topic, target_accts):
         subscribe_permissions = iam.PolicyStatement()
-        subscribe_permissions.add_arn_principal(f'arn:aws:iam::{Aws.ACCOUNT_ID}:root')
-        for id in account_ids:
-            subscribe_permissions.add_arn_principal(f'arn:aws:iam::{id}:root')
+        subscribe_permissions.add_arn_principal(f"arn:aws:iam::{Aws.ACCOUNT_ID}:root")
+        for language in target_accts.keys():
+            if "enabled" in str(target_accts[language]["status"]):
+                subscribe_permissions.add_arn_principal(
+                    f"arn:aws:iam::{str(target_accts[language]['account_id'])}:root"
+                )
         subscribe_permissions.add_actions("SNS:Subscribe")
         subscribe_permissions.add_resources(topic.topic_arn)
         topic.add_to_resource_policy(subscribe_permissions)
 
-        # Set up cross-account Publish permissions for every onboarded language.
+    def init_publish_permissions(self, topic, target_accts):
         publish_permissions = iam.PolicyStatement()
-        publish_permissions.add_arn_principal(f'arn:aws:iam::{Aws.ACCOUNT_ID}:root')
-        for id in account_ids:
-            subscribe_permissions.add_arn_principal(f'arn:aws:iam::{id}:root')
+        publish_permissions.add_arn_principal(f"arn:aws:iam::{Aws.ACCOUNT_ID}:root")
+        for language in target_accts.keys():
+            publish_permissions.add_arn_principal(
+                f"arn:aws:iam::{str(target_accts[language]['account_id'])}:root"
+            )
         publish_permissions.add_actions("SNS:Publish")
         publish_permissions.add_service_principal("events.amazonaws.com")
         publish_permissions.add_resources(topic.topic_arn)
         topic.add_to_resource_policy(publish_permissions)
+
+    def init_create_bucket(self, bucket_name):
+        bucket = s3.Bucket(
+            self,
+            bucket_name,
+            bucket_name=bucket_name,
+            versioned=False,
+            block_public_access=s3.BlockPublicAccess.BLOCK_ALL,
+        )
+        return bucket
+
+    def init_cross_account_log_role(self, target_accts, bucket):
+        languages = target_accts.keys()
+        if len(languages) > 0:
+            # Define policy that allows cross-account Amazon SNS and Amazon SQS access.
+            statement = iam.PolicyStatement()
+            statement.add_actions("s3:PutObject", "s3:PutObjectAcl")
+            statement.add_resources(f"{bucket.bucket_arn}/*")
+            for language in languages:
+                if "enabled" in str(target_accts[language]["status"]):
+                    statement.add_arn_principal(
+                        f"arn:aws:iam::{str(target_accts[language]['account_id'])}:role/LogsLambdaExecutionRole"
+                    )
+            bucket.add_to_resource_policy(statement)
diff --git a/test/eventbridge_rule_with_sns_fanout/requirements.txt b/test/eventbridge_rule_with_sns_fanout/requirements.txt
@@ -1,4 +1,7 @@
+boto3
 aws-cdk-lib>=2.0.0
 constructs>=10.0.0
 types-boto3
 types-setuptools
+random
+pyyaml
diff --git a/test/eventbridge_rule_with_sns_fanout/store_system_parameters.py b/test/eventbridge_rule_with_sns_fanout/store_system_parameters.py
diff --git a/test/public_ecr_repositories/app.py b/test/public_ecr_repositories/app.py
@@ -2,12 +2,16 @@
 import os
 
 import aws_cdk as cdk
-
-from public_ecr_repositories_stack.public_ecr_repositories_stack import PublicEcrRepositoriesStack
+from public_ecr_repositories_stack.public_ecr_repositories_stack import \
+    PublicEcrRepositoriesStack
 
 app = cdk.App()
-PublicEcrRepositoriesStack(app, "PublicEcrRepositoriesStack",
-    env=cdk.Environment(account=os.getenv('CDK_DEFAULT_ACCOUNT'), region=os.getenv('CDK_DEFAULT_REGION')),
-    )
+PublicEcrRepositoriesStack(
+    app,
+    "PublicEcrRepositoriesStack",
+    env=cdk.Environment(
+        account=os.getenv("CDK_DEFAULT_ACCOUNT"), region=os.getenv("CDK_DEFAULT_REGION")
+    ),
+)
 
 app.synth()
diff --git a/test/public_ecr_repositories/public_ecr_repositories_stack/public_ecr_repositories_stack.py b/test/public_ecr_repositories/public_ecr_repositories_stack/public_ecr_repositories_stack.py
@@ -1,38 +1,30 @@
-from aws_cdk import (
-    Stack,
-    aws_ecr as ecr
-)
+from aws_cdk import Stack
+from aws_cdk import aws_ecr as ecr
 from constructs import Construct
+import yaml
 
 class PublicEcrRepositoriesStack(Stack):
-
     def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
         super().__init__(scope, construct_id, **kwargs)
 
-        languages = [
-            'ruby',
-            'javav2',
-            'javascriptv3',
-            'gov2',
-            'python',
-            'dotnetv3',
-            'kotlin',
-            'rust_dev_preview',
-            'swift',
-            'cpp',
-            'sap-abap'
-        ]
-
-        for language in languages:
+        target_accts = self.get_yaml_config("../../config/targets.yaml")
 
-            usage_text = f'This image provides a pre-built SDK for {language} environment and is recommended for local testing of SDK for {language} example code. It is not intended for production usage. For detailed and up-to-date steps on running this image, see https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/{language}/README.md#docker-image-beta.'
+        for language in target_accts.keys():
+            usage_text = f"This image provides a pre-built SDK for {language} environment and is recommended for local testing of SDK for {language} example code. It is not intended for production usage. For detailed and up-to-date steps on running this image, see https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/{language}/README.md#docker-image-beta."
             repository_description = f'This image provides a pre-built for SDK for {language} environment and is recommended for local testing of SDK for {language} example code."'
-            ecr.CfnPublicRepository(self, f"{language}",
+            ecr.CfnPublicRepository(
+                self,
+                f"{language}",
                 repository_name=language,
                 repository_catalog_data={
                     "UsageText": usage_text,
                     "OperatingSystems": ["Linux"],
                     "Architectures": ["x86", "ARM"],
-                    "RepositoryDescription": repository_description
-                }
+                    "RepositoryDescription": repository_description,
+                },
             )
+
+    def get_yaml_config(self, filepath):
+        with open(filepath, "r") as file:
+            data = yaml.safe_load(file)
+        return data