diff --git a/.github/linters/checkstyle.xml b/.github/linters/checkstyle.xml index 0bd4786a01a..ad3362bdd98 100644 --- a/.github/linters/checkstyle.xml +++ b/.github/linters/checkstyle.xml @@ -1,25 +1,21 @@ - - - - - - + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + \ No newline at end of file diff --git a/.github/pre_validate/.gitignore b/.github/pre_validate/.gitignore deleted file mode 100644 index bee8a64b79a..00000000000 --- a/.github/pre_validate/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__ diff --git a/.github/pre_validate/words.py b/.github/pre_validate/words.py deleted file mode 100644 index 1ef93416ee6..00000000000 --- a/.github/pre_validate/words.py +++ /dev/null @@ -1,5 +0,0 @@ -from urllib.request import urlopen -import json -GOODWORDS={"throat", "dummy", "dp", "jerry"} -DATA=urlopen("https://raw.githubusercontent.com/zacanger/profane-words/5ad6c62fa5228293bc610602eae475d50036dac2/words.json") -WORDS=set(json.load(DATA)).difference(GOODWORDS) \ No newline at end of file diff --git a/.github/workflows/pre-validate.yml b/.github/workflows/pre-validate.yml index 0287eae0458..c30f4398658 100644 --- a/.github/workflows/pre-validate.yml +++ b/.github/workflows/pre-validate.yml @@ -5,9 +5,9 @@ name: Miscellaneous pre-validation checks on: # Triggers the workflow on push or pull request events but only for the master branch push: - branches: [ main ] + branches: [main] pull_request: - branches: [ main ] + branches: [main] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -29,4 +29,4 @@ jobs: python-version: 3.8 #install the python needed - name: Run Python script to vet code examples # Runs a single command using the runners shell run: | - python .github/pre_validate/pre_validate.py --q + python3 .tools/pre_validate/pre_validate.py --q diff --git a/.github/workflows/validate-doc-metadata.yml b/.github/workflows/validate-doc-metadata.yml index 07bfcde223a..acb5cf77930 100644 --- a/.github/workflows/validate-doc-metadata.yml +++ b/.github/workflows/validate-doc-metadata.yml @@ -20,8 +20,8 @@ jobs: python -m pip install yamale python -m pip install yamllint - name: Lint with yamllint - run: yamllint --format standard -c .tools/validation/.yamllint.yaml .doc_gen/metadata/. + run: yamllint --format standard -c .tools/validation/.yamllint.yaml .doc_gen/metadata - name: Lint with yamllint - run: yamllint --format standard -c .tools/validation/.yamllint.yaml .doc_gen/metadata/curated/. + run: yamllint --format standard -c .tools/validation/.yamllint.yaml .doc_gen/metadata/curated - name: Validate metadata with yamale - run: python .tools/validation/validate_doc_metadata.py .doc_gen + run: python .tools/validation/validate_doc_metadata.py --doc-gen .doc_gen diff --git a/.tools/readmes/.gitignore b/.tools/.gitignore similarity index 78% rename from .tools/readmes/.gitignore rename to .tools/.gitignore index d116609c0eb..9f7550b1ef0 100644 --- a/.tools/readmes/.gitignore +++ b/.tools/.gitignore @@ -1,3 +1,2 @@ __pycache__ .venv -venv diff --git a/.tools/images/render-blurbs.py b/.tools/images/render-blurbs.py index 9a5ceb497a4..b5494934a70 100644 --- a/.tools/images/render-blurbs.py +++ b/.tools/images/render-blurbs.py @@ -16,7 +16,6 @@ with open(sdk_metadata, "r") as file: metadata = yaml.safe_load(file) for language in metadata.keys(): - metadata[language] shortname = metadata[language]["property"] template = env.get_template("template.txt") print(template.render(language=language, shortname=shortname)) diff --git a/.github/pre_validate/README.md b/.tools/pre_validate/README.md similarity index 100% rename from .github/pre_validate/README.md rename to .tools/pre_validate/README.md diff --git a/.github/pre_validate/pre_validate.py b/.tools/pre_validate/pre_validate.py similarity index 97% rename from .github/pre_validate/pre_validate.py rename to .tools/pre_validate/pre_validate.py index 157510c6d66..445cd1003a2 100644 --- a/.github/pre_validate/pre_validate.py +++ b/.tools/pre_validate/pre_validate.py @@ -59,25 +59,26 @@ # Folders to skip. IGNORE_FOLDERS = { + ".doc_gen", ".pytest_cache", + ".tools", + ".venv", "__pycache__", + "bin", "cdk.out", - "node_modules", "dist", + "node_modules", + "obj", "target", "venv", - ".venv", - "bin", - "obj", - ".doc_gen", } # Files to skip. IGNORE_FILES = { - "AssemblyInfo.cs", + ".moviedata.json", ".travis.yml", + "AssemblyInfo.cs", "moviedata.json", - ".moviedata.json", "movies.json", "package-lock.json", } @@ -85,14 +86,14 @@ # Sample files. EXPECTED_SAMPLE_FILES = { "README.md", - "movies.json", - "speech_sample.mp3", "chat_sfn_state_machine.json", "market_2.jpg", - "spheres_2.jpg", + "movies.json", "sample_cert.pem", "sample_private_key.pem", "sample_saml_metadata.xml", + "speech_sample.mp3", + "spheres_2.jpg", } # Media file types. @@ -109,13 +110,35 @@ "AKIAEXAMPLEACCESSKEY", "AKIAIOSFODNN7EXAMPLE", "APKAEIBAERJR2EXAMPLE", + "AWSEC2/latest/APIReference/OperationList", "AppStreamUsageReportsCFNGlueAthenaAccess", - "examples/blob/main/applications/feedback", + "CancelExportTaskExample/CancelExportTask", + "CertificateTransparencyLoggingPreference", + "ChangeMessageVisibilityBatchRequestEntry", + "CreateCollectionExample/CreateCollection", + "CreateExportTaskExample/CreateExportTask", + "DeleteCollectionExample/DeleteCollection", + "DescribeDbClusterParameterGroupsResponse", + "DescribeOrderableDBInstanceOptionsOutput", + "DynamodbRubyExampleCreateUsersTableStack", + "GetIdentityVerificationAttributesRequest", + "KMSWithContextEncryptionMaterialsExample", + "KinesisStreamSourceConfiguration=kinesis", + "ListOrganizationalUnitsForParentResponse", + "ListTagsExample/ListTagsExample/ListTags", + "ListTagsForVaultExample/ListTagsForVault", + "SynthesizeSpeechExample/SynthesizeSpeech", + "TargetTrackingScalingPolicyConfiguration", + "TerminateInstanceInAutoScalingGroupAsync", + "VectorEnrichmentJobDataSourceConfigInput", + "amazondynamodb/latest/developerguide/DAX", + "apigateway/latest/developerguide/welcome", "aws/acm/model/DescribeCertificateRequest", "aws/cloudtrail/model/LookupEventsRequest", "aws/codebuild/model/BatchGetBuildsResult", "aws/codecommit/model/DeleteBranchRequest", "aws/codecommit/model/ListBranchesRequest", + "aws/dynamodb/model/BatchWriteItemRequest", "aws/dynamodb/model/ProvisionedThroughput", "aws/ec2/model/CreateSecurityGroupRequest", "aws/ec2/model/DeleteSecurityGroupRequest", @@ -129,87 +152,58 @@ "aws/iam/model/GetServerCertificateResult", "aws/kinesis/model/GetShardIteratorResult", "aws/kinesis/model/PutRecordsRequestEntry", + "aws/kms/model/ScheduleKeyDeletionRequest", "aws/monitoring/model/DeleteAlarmsRequest", "aws/neptune/model/CreateDBClusterRequest", "aws/neptune/model/DeleteDBClusterRequest", "aws/neptune/model/ModifyDBClusterRequest", - "aws/kms/model/ScheduleKeyDeletionRequest", - "KMSWithContextEncryptionMaterialsExample", - "CertificateTransparencyLoggingPreference", - "ChangeMessageVisibilityBatchRequestEntry", + "aws/rds/model/DescribeDBInstancesRequest", + "aws/rds/model/DescribeDBSnapshotsRequest", + "cloudwatch/commands/PutMetricDataCommand", + "com/amazondynamodb/latest/developerguide", + "com/apigateway/latest/developerguide/set", + "com/autoscaling/ec2/APIReference/Welcome", + "com/awssupport/latest/APIReference/index", + "com/firehose/latest/APIReference/Welcome", "com/greengrass/latest/developerguide/lra", "com/greengrass/latest/developerguide/sns", + "com/pinpoint/latest/apireference/welcome", + "com/redshift/latest/APIReference/Welcome", + "com/rekognition/latest/dg/considerations", "com/samples/JobStatusNotificationsSample", + "com/transcribe/latest/APIReference/index", + "com/v1/documentation/api/latest/guide/s3", + "com/workdocs/latest/APIReference/Welcome", + "devicefarm/latest/developerguide/welcome", + "examples/blob/main/applications/feedback", "generate_presigned_url_and_upload_object", - "KinesisStreamSourceConfiguration=kinesis", - "ListOrganizationalUnitsForParentResponse", + "iam/commands/GetAccessKeyLastUsedCommand", + "iam/commands/GetServerCertificateCommand", "nFindProductsWithNegativePriceWithConfig", + "preview/examples/cognitoidentityprovider", + "preview/examples/cognitoidentityprovider", + "preview/examples/lambda/src/bin/scenario", + "role/AmazonSageMakerGeospatialFullAccess", "s3_client_side_encryption_sym_master_key", "serial/CORE_THING_NAME/write/dev/serial1", - "TargetTrackingScalingPolicyConfiguration", - "targetTrackingScalingPolicyConfiguration", - "upload_files_using_managed_file_uploader", - "videoMetaData=celebrityRecognitionResult", - "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - "com/v1/documentation/api/latest/guide/s3", - "iam/commands/GetServerCertificateCommand", - "iam/commands/GetAccessKeyLastUsedCommand", - "iam/commands/GetAccessKeyLastUsedCommand", - "iam/commands/GetServerCertificateCommand", - "cloudwatch/commands/PutMetricDataCommand", - "ses/commands/VerifyDomainIdentityCommand", - "ses/commands/DeleteReceiptRuleSetCommand", - "ses/commands/DeleteReceiptRuleSetCommand", + "service/FeedbackSentimentAnalyzer/README", "ses/commands/CreateReceiptRuleSetCommand", + "ses/commands/DeleteReceiptRuleSetCommand", "ses/commands/VerifyDomainIdentityCommand", - "ses/commands/VerifyDomainIdentityCommand", - "com/amazondynamodb/latest/developerguide", - "DynamodbRubyExampleCreateUsersTableStack", - "com/rekognition/latest/dg/considerations", - "ListTagsForVaultExample/ListTagsForVault", - "TerminateInstanceInAutoScalingGroupAsync", - "GetIdentityVerificationAttributesRequest", - "com/transcribe/latest/APIReference/index", - "CancelExportTaskExample/CancelExportTask", - "CreateExportTaskExample/CreateExportTask", - "ListTagsExample/ListTagsExample/ListTags", - "SynthesizeSpeechExample/SynthesizeSpeech", - "com/autoscaling/ec2/APIReference/Welcome", - "CreateCollectionExample/CreateCollection", - "DeleteCollectionExample/DeleteCollection", - "com/awssupport/latest/APIReference/index", - "DescribeDbClusterParameterGroupsResponse", - "DeleteCollectionExample/DeleteCollection", - "aws/rds/model/DescribeDBInstancesRequest", - "com/apigateway/latest/developerguide/set", - "com/pinpoint/latest/apireference/welcome", - "src/main/java/com/example/s3/S3BucketOps", - "src/main/java/com/example/s3/ListObjects", - "amazondynamodb/latest/developerguide/DAX", - "apigateway/latest/developerguide/welcome", - "devicefarm/latest/developerguide/welcome", - "AWSEC2/latest/APIReference/OperationList", "src/main/java/com/example/dynamodb/Query", - "com/firehose/latest/APIReference/Welcome", - "src/main/java/com/example/iam/CreateRole", "src/main/java/com/example/iam/CreateUser", "src/main/java/com/example/iam/DeleteUser", "src/main/java/com/example/iam/UpdateUser", "src/main/java/com/example/kms/ListGrants", - "com/redshift/latest/APIReference/Welcome", + "src/main/java/com/example/s3/ListObjects", + "src/main/java/com/example/s3/S3BucketOps", "src/main/java/com/example/sns/ListOptOut", "src/main/java/com/example/sns/ListTopics", "src/main/java/com/example/sqs/SQSExample", - "DescribeOrderableDBInstanceOptionsOutput", - "preview/examples/cognitoidentityprovider", - "preview/examples/lambda/src/bin/scenario", - "aws/dynamodb/model/BatchWriteItemRequest", - "aws/rds/model/DescribeDBInstancesRequest", - "aws/rds/model/DescribeDBSnapshotsRequest", - "role/AmazonSageMakerGeospatialFullAccess", - "VectorEnrichmentJobDataSourceConfigInput", - "com/workdocs/latest/APIReference/Welcome", - "service/FeedbackSentimentAnalyzer/README", + "targetTrackingScalingPolicyConfiguration", + "upload_files_using_managed_file_uploader", + "videoMetaData=celebrityRecognitionResult", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", } diff --git a/.github/pre_validate/test/test_pre_validate.py b/.tools/pre_validate/test/test_pre_validate.py similarity index 100% rename from .github/pre_validate/test/test_pre_validate.py rename to .tools/pre_validate/test/test_pre_validate.py diff --git a/.tools/pre_validate/words.py b/.tools/pre_validate/words.py new file mode 100644 index 00000000000..481edfaccd7 --- /dev/null +++ b/.tools/pre_validate/words.py @@ -0,0 +1,8 @@ +from urllib.request import urlopen +import json + +GOOD_WORDS = {"throat", "dummy", "dp", "jerry"} +DATA = urlopen( + "https://raw.githubusercontent.com/zacanger/profane-words/5ad6c62fa5228293bc610602eae475d50036dac2/words.json" +) +WORDS = set(json.load(DATA)).difference(GOOD_WORDS) diff --git a/.tools/validation/schema/.yamllint.yaml b/.tools/validation/.yamllint.yaml similarity index 100% rename from .tools/validation/schema/.yamllint.yaml rename to .tools/validation/.yamllint.yaml diff --git a/.tools/validation/validate_doc_metadata.py b/.tools/validation/validate_doc_metadata.py index d901888bd8b..b781fbe2520 100755 --- a/.tools/validation/validate_doc_metadata.py +++ b/.tools/validation/validate_doc_metadata.py @@ -167,24 +167,15 @@ def validate_files(schema_name, meta_names, validators): return success -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--doc_gen", - default=f"{Path(__file__).parent / '..' / '..' / '.doc_gen'}", - help="The folder that contains schema and metadata files.", - required=False, - ) - args = parser.parse_args() - +def validate_all(doc_gen: Path): # with open(os.path.join(args.doc_gen, "metadata/sdks.yaml")) as sdks_file: # sdks_yaml: dict[str, any] = yaml.safe_load(sdks_file) - with open(os.path.join(args.doc_gen, "metadata/services.yaml")) as services_file: + with open(os.path.join(doc_gen, "metadata/services.yaml")) as services_file: services_yaml = yaml.safe_load(services_file) with open( - os.path.join(args.doc_gen, "metadata/curated/sources.yaml") + os.path.join(doc_gen, "metadata/curated/sources.yaml") ) as curated_sources_file: curated_sources_yaml = yaml.safe_load(curated_sources_file) @@ -192,7 +183,7 @@ def main(): ServiceName.services = services_yaml SourceKey.curated_sources = curated_sources_yaml ExampleId.services = services_yaml - BlockContent.block_names = os.listdir(os.path.join(args.doc_gen, "cross-content")) + BlockContent.block_names = os.listdir(os.path.join(doc_gen, "cross-content")) validators[ServiceName.tag] = ServiceName validators[ServiceVersion.tag] = ServiceVersion validators[SourceKey.tag] = SourceKey @@ -204,31 +195,42 @@ def main(): # Validate sdks.yaml file. schema_name = schema_root / "sdks_schema.yaml" - meta_names = glob.glob(os.path.join(args.doc_gen, "metadata/sdks.yaml")) + meta_names = glob.glob(os.path.join(doc_gen, "metadata/sdks.yaml")) success = validate_files(schema_name, meta_names, validators) # Validate services.yaml file. schema_name = schema_root / "services_schema.yaml" - meta_names = glob.glob(os.path.join(args.doc_gen, "metadata/services.yaml")) + meta_names = glob.glob(os.path.join(doc_gen, "metadata/services.yaml")) success &= validate_files(schema_name, meta_names, validators) # Validate example (*_metadata.yaml in metadata folder) files. schema_name = schema_root / "example_schema.yaml" - meta_names = glob.glob(os.path.join(args.doc_gen, "metadata/*_metadata.yaml")) + meta_names = glob.glob(os.path.join(doc_gen, "metadata/*_metadata.yaml")) success &= validate_files(schema_name, meta_names, validators) # Validate curated/sources.yaml file. schema_name = schema_root / "curated_sources_schema.yaml" - meta_names = glob.glob(os.path.join(args.doc_gen, "metadata/curated/sources.yaml")) + meta_names = glob.glob(os.path.join(doc_gen, "metadata/curated/sources.yaml")) success &= validate_files(schema_name, meta_names, validators) # Validate curated example (*_metadata.yaml in metadata/curated folder) files. schema_name = schema_root / "curated_example_schema.yaml" - meta_names = glob.glob( - os.path.join(args.doc_gen, "metadata/curated/*_metadata.yaml") - ) + meta_names = glob.glob(os.path.join(doc_gen, "metadata/curated/*_metadata.yaml")) success &= validate_files(schema_name, meta_names, validators) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--doc-gen", + default=f"{Path(__file__).parent / '..' / '..' / '.doc_gen'}", + help="The folder that contains schema and metadata files.", + required=False, + ) + args = parser.parse_args() + + success = validate_all(args.doc_gen) + if success: print("Validation succeeded! 👍👍👍") else: