diff --git a/.doc_gen/images/render-blurbs.py b/.doc_gen/images/render-blurbs.py deleted file mode 100644 index c00fa25740d..00000000000 --- a/.doc_gen/images/render-blurbs.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -import jinja2 -import yaml - -env = jinja2.Environment( - loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), - autoescape=jinja2.select_autoescape( - enabled_extensions=("html", "xml"), default_for_string=True - ), -) - -with open("../metadata/sdks.yaml", "r") as file: - metadata = yaml.safe_load(file) - for language in metadata.keys(): - metadata[language] - shortname = metadata[language]["property"] - template = env.get_template("template.txt") - print(template.render(language=language, shortname=shortname)) diff --git a/.doc_gen/readmes/writeme.sh b/.doc_gen/readmes/writeme.sh deleted file mode 100644 index 82e743c119c..00000000000 --- a/.doc_gen/readmes/writeme.sh +++ /dev/null @@ -1,83 +0,0 @@ -SVCS=(acm -api-gateway -apigatewaymanagementapi -application-autoscaling -auditmanager -aurora -auto-scaling -batch -cloudformation -cloudfront -cloudwatch -cloudwatch-events -cloudwatch-logs -cognito -cognito-identity -cognito-identity-provider -cognito-sync -comprehend -config-service -device-farm -dynamodb -ebs -ec2 -ecr -ecs -eks -emr -eventbridge -firehose -glacier -glue -iam -iot -keyspaces -kinesis -kinesis-analytics-v2 -kms -lambda -lex -lookoutvision -mediaconvert -medialive -mediapackage -organizations -personalize -personalize-runtime -personalize-events -pinpoint -pinpoint-email -pinpoint-sms-voice -polly -qldb -rds -rds-data -redshift -rekognition -route-53 -route53-domains -route53-recovery-cluster -s3 -sagemaker -secrets-manager -ses -sesv2 -sfn -snowball -sns -sqs -ssm -sts -support -textract -transcribe -transcribe-medical -translate) - -for SVC in ${SVCS[@]} ; do - F=rust_dev_preview/${SVC/-/} - if [ -d $F ] ; then - ./.doc_gen/readmes/venv/bin/python ./.doc_gen/readmes/writeme.py \ - --svc_folder $F Rust 1 $SVC - fi -done \ No newline at end of file diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000000..772c0c99da7 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +# In the root for editors when opening the workspace as a whole. +[flake8] +extend-ignore = E203,E501 +exclude = .git,__pycache__, \ No newline at end of file diff --git a/.tools/.flake8 b/.tools/.flake8 new file mode 100644 index 00000000000..a6352e420a9 --- /dev/null +++ b/.tools/.flake8 @@ -0,0 +1,5 @@ +[flake8] +extend-ignore = E203,E501 +exclude = + .git, + __pycache__, \ No newline at end of file diff --git a/.tools/README.md b/.tools/README.md new file mode 100644 index 00000000000..e34fcb5db43 --- /dev/null +++ b/.tools/README.md @@ -0,0 +1,84 @@ +# SDK Code Examples Tools + +These tools help SDK Code Example developers' integration with internal docs tooling. +This is mainly in the form of .doc_gen SOS metadata validation, with other tools as necessary. + +SDK Code Examples tools team uses Python for our tools because of its cross-platform runtime and broad knowledge base. + +## Tooling Guidelines + +1. Use Python latest for tooling (this is not the same rule as for python sdk, be aware). +2. Our “standard library” adds `flake8`, `black`, `pyyaml`, `yamale`, and `requests` + - Current versions are in base_requirements.txt +3. Run `.tools/verify_python.py` to ensure python tools is configured as expected. + - This will install our “standard library” to site_packages and ensure that the python version is at least 3.(current - 1). +4. Use a venv in `.venv` if the project needs packages beyond our standard library. + 1. Include those libraries in the `requirements.txt`. + 2. Copy `.tools/base-requirements.txt` to the tool’s folder to seed `requirements.txt` if you like + 3. With an active venv, run `pip freeze > requirements.txt` to update the tool’s requirements. +5. Keep tools self executable + 1. Mark them executable (`chmod a+x ` in \*nix) + 2. Add a shebang `#!/usr/bin/env python3` + 3. Use `__name__ == “__main__”` check that delegates to a `main` function +6. Use f-strings for string building. +7. Use `logging` to write diagnostics to the console. +8. Format using Black with default settings. +9. Lint using `flake8` and default settings. + - Treat Warnings as errors + - Except line length E501, because it disagrees with black +10. Verify type annotations using mypy. + - Type annotations are strongly recommended. +11. Run tests using pytest. +12. Parse arguments using argparse. + - All scripts must run headless, without user interaction. +13. Use pathlib for files and paths. + - Prefer os.scandir to os.listdir. +14. Parse & dump yaml using PyYAML. (imports from `yaml`) + - Validate yaml schemas using `yamale` +15. When working with dates, import from datetime and always include a timezone (usually `timezone.utc`). +16. Prefer data classes or immutable tuples for base data types. +17. Use requests for web calls, but prefer a native wrapper (like boto3 or pygithub) when available. + - If you http/2, you might consider httpx +18. Use subprocess for system calls, but prefer a native wrapper (like gitpython) when available. + +## Some PEPs you might want to know about: + +Python 3.12: + +- [PEP 701](http://www.python.org/dev/peps/pep-0701) -- More flexible f-string parsing +- [PEP 695](http://www.python.org/dev/peps/pep-0695) -- New type annotations syntax for generic classes + +Python 3.11: + +- [PEP 654](http://www.python.org/dev/peps/pep-0654) -- Exception Groups and except\* + +Python 3.10: + +- [PEP 604](http://www.python.org/dev/peps/pep-0604) -- Allow writing union types as X | Y +- [PEP 636](http://www.python.org/dev/peps/pep-0636) -- Structural Pattern Matching: Tutorial + +Python 3.9: + +- [PEP 585](http://www.python.org/dev/peps/pep-0585), Type Hinting Generics In Standard Collections +- [PEP 602](http://www.python.org/dev/peps/pep-0602), Python adopts a stable annual release cadence + +Python 3.8 + +- [PEP 572](http://www.python.org/dev/peps/pep-0572), Assignment expressions +- [PEP 586](http://www.python.org/dev/peps/pep-0586), Literal types +- [PEP 589](http://www.python.org/dev/peps/pep-0589), TypedDict + +Python 3.7 + +- [PEP 557](http://www.python.org/dev/peps/pep-0557), Data Classes + +## VSCode Extensions + +- https://marketplace.visualstudio.com/items?itemName=ms-python.mypy-type-checker +- https://marketplace.visualstudio.com/items?itemName=ms-python.flake8 +- https://marketplace.visualstudio.com/items?itemName=ms-python.black-formatter + +## Flake8 Ignore + +- E203 whitespace before ':' +- E501 line too long diff --git a/.tools/base_requirements.txt b/.tools/base_requirements.txt new file mode 100644 index 00000000000..6a164f18ed1 --- /dev/null +++ b/.tools/base_requirements.txt @@ -0,0 +1,18 @@ +black==23.9.1 +certifi==2023.7.22 +charset-normalizer==3.3.0 +click==8.1.7 +flake8==6.1.0 +idna==3.4 +mccabe==0.7.0 +mypy-extensions==1.0.0 +packaging==23.2 +pathspec==0.11.2 +platformdirs==3.11.0 +pycodestyle==2.11.1 +pyflakes==3.1.0 +PyYAML==6.0.1 +requests==2.31.0 +types-PyYAML==6.0.12.12 +urllib3==2.0.6 +yamale==4.0.4 diff --git a/.doc_gen/images/README.md b/.tools/images/README.md similarity index 100% rename from .doc_gen/images/README.md rename to .tools/images/README.md diff --git a/.doc_gen/images/__init__.py b/.tools/images/__init__.py similarity index 100% rename from .doc_gen/images/__init__.py rename to .tools/images/__init__.py diff --git a/.tools/images/render-blurbs.py b/.tools/images/render-blurbs.py new file mode 100644 index 00000000000..9a5ceb497a4 --- /dev/null +++ b/.tools/images/render-blurbs.py @@ -0,0 +1,22 @@ +import os + +import jinja2 +import yaml +from pathlib import Path + +env = jinja2.Environment( + loader=jinja2.FileSystemLoader(os.path.dirname(__file__)), + autoescape=jinja2.select_autoescape( + enabled_extensions=("html", "xml"), default_for_string=True + ), +) + +if __name__ == "__main__": + sdk_metadata = Path(__file__).parent / ".." / ".." / "metadata" / "sdks.yaml" + with open(sdk_metadata, "r") as file: + metadata = yaml.safe_load(file) + for language in metadata.keys(): + metadata[language] + shortname = metadata[language]["property"] + template = env.get_template("template.txt") + print(template.render(language=language, shortname=shortname)) diff --git a/.doc_gen/images/template.txt b/.tools/images/template.txt similarity index 100% rename from .doc_gen/images/template.txt rename to .tools/images/template.txt diff --git a/.doc_gen/readmes/.gitignore b/.tools/readmes/.gitignore similarity index 100% rename from .doc_gen/readmes/.gitignore rename to .tools/readmes/.gitignore diff --git a/.doc_gen/readmes/README.md b/.tools/readmes/README.md similarity index 100% rename from .doc_gen/readmes/README.md rename to .tools/readmes/README.md diff --git a/.doc_gen/readmes/config.py b/.tools/readmes/config.py similarity index 100% rename from .doc_gen/readmes/config.py rename to .tools/readmes/config.py diff --git a/.doc_gen/readmes/includes/code_examples.jinja2 b/.tools/readmes/includes/code_examples.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/code_examples.jinja2 rename to .tools/readmes/includes/code_examples.jinja2 diff --git a/.doc_gen/readmes/includes/copyright.jinja2 b/.tools/readmes/includes/copyright.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/copyright.jinja2 rename to .tools/readmes/includes/copyright.jinja2 diff --git a/.doc_gen/readmes/includes/important.jinja2 b/.tools/readmes/includes/important.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/important.jinja2 rename to .tools/readmes/includes/important.jinja2 diff --git a/.doc_gen/readmes/includes/macros.jinja2 b/.tools/readmes/includes/macros.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/macros.jinja2 rename to .tools/readmes/includes/macros.jinja2 diff --git a/.doc_gen/readmes/includes/overview.jinja2 b/.tools/readmes/includes/overview.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/overview.jinja2 rename to .tools/readmes/includes/overview.jinja2 diff --git a/.doc_gen/readmes/includes/prerequisites.jinja2 b/.tools/readmes/includes/prerequisites.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/prerequisites.jinja2 rename to .tools/readmes/includes/prerequisites.jinja2 diff --git a/.doc_gen/readmes/includes/resources.jinja2 b/.tools/readmes/includes/resources.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/resources.jinja2 rename to .tools/readmes/includes/resources.jinja2 diff --git a/.doc_gen/readmes/includes/run_instructions.jinja2 b/.tools/readmes/includes/run_instructions.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/run_instructions.jinja2 rename to .tools/readmes/includes/run_instructions.jinja2 diff --git a/.doc_gen/readmes/includes/tests.jinja2 b/.tools/readmes/includes/tests.jinja2 similarity index 100% rename from .doc_gen/readmes/includes/tests.jinja2 rename to .tools/readmes/includes/tests.jinja2 diff --git a/.doc_gen/readmes/multi.py b/.tools/readmes/multi.py similarity index 100% rename from .doc_gen/readmes/multi.py rename to .tools/readmes/multi.py diff --git a/.doc_gen/readmes/render.py b/.tools/readmes/render.py similarity index 100% rename from .doc_gen/readmes/render.py rename to .tools/readmes/render.py diff --git a/.doc_gen/readmes/requirements.txt b/.tools/readmes/requirements.txt similarity index 100% rename from .doc_gen/readmes/requirements.txt rename to .tools/readmes/requirements.txt diff --git a/.doc_gen/readmes/scanner.py b/.tools/readmes/scanner.py similarity index 100% rename from .doc_gen/readmes/scanner.py rename to .tools/readmes/scanner.py diff --git a/.doc_gen/readmes/service_readme.jinja2 b/.tools/readmes/service_readme.jinja2 similarity index 100% rename from .doc_gen/readmes/service_readme.jinja2 rename to .tools/readmes/service_readme.jinja2 diff --git a/.doc_gen/readmes/snippets.py b/.tools/readmes/snippets.py similarity index 100% rename from .doc_gen/readmes/snippets.py rename to .tools/readmes/snippets.py diff --git a/.doc_gen/readmes/writeme.py b/.tools/readmes/writeme.py similarity index 100% rename from .doc_gen/readmes/writeme.py rename to .tools/readmes/writeme.py diff --git a/.tools/validation/.flake8 b/.tools/validation/.flake8 new file mode 100644 index 00000000000..a6352e420a9 --- /dev/null +++ b/.tools/validation/.flake8 @@ -0,0 +1,5 @@ +[flake8] +extend-ignore = E203,E501 +exclude = + .git, + __pycache__, \ No newline at end of file diff --git a/.doc_gen/validation/.yamllint.yaml b/.tools/validation/schema/.yamllint.yaml similarity index 100% rename from .doc_gen/validation/.yamllint.yaml rename to .tools/validation/schema/.yamllint.yaml diff --git a/.doc_gen/validation/curated_example_schema.yaml b/.tools/validation/schema/curated_example_schema.yaml similarity index 100% rename from .doc_gen/validation/curated_example_schema.yaml rename to .tools/validation/schema/curated_example_schema.yaml diff --git a/.doc_gen/validation/curated_sources_schema.yaml b/.tools/validation/schema/curated_sources_schema.yaml similarity index 100% rename from .doc_gen/validation/curated_sources_schema.yaml rename to .tools/validation/schema/curated_sources_schema.yaml diff --git a/.doc_gen/validation/example_schema.yaml b/.tools/validation/schema/example_schema.yaml similarity index 100% rename from .doc_gen/validation/example_schema.yaml rename to .tools/validation/schema/example_schema.yaml diff --git a/.doc_gen/validation/sdks_schema.yaml b/.tools/validation/schema/sdks_schema.yaml similarity index 100% rename from .doc_gen/validation/sdks_schema.yaml rename to .tools/validation/schema/sdks_schema.yaml diff --git a/.doc_gen/validation/services_schema.yaml b/.tools/validation/schema/services_schema.yaml similarity index 100% rename from .doc_gen/validation/services_schema.yaml rename to .tools/validation/schema/services_schema.yaml diff --git a/.doc_gen/validation/validate_doc_metadata.py b/.tools/validation/validate_doc_metadata.py old mode 100644 new mode 100755 similarity index 55% rename from .doc_gen/validation/validate_doc_metadata.py rename to .tools/validation/validate_doc_metadata.py index 143e64a137f..d901888bd8b --- a/.doc_gen/validation/validate_doc_metadata.py +++ b/.tools/validation/validate_doc_metadata.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 """ Validator for mapping and example metadata used to generate code example documentation. This validator uses Yamale (https://github.com/23andMe/Yamale) to compare a schema @@ -11,34 +12,36 @@ import re import yaml import yamale +from pathlib import Path from yamale import YamaleError from yamale.validators import DefaultValidators, Validator, String class ServiceName(Validator): - """ Validate that service names appear in services.yaml. """ - tag = 'service_name' + """Validate that service names appear in services.yaml.""" + + tag = "service_name" services = {} def get_name(self): - return 'service name found in services.yaml' + return "service name found in services.yaml" def _is_valid(self, value): return value in self.services class ServiceVersion(Validator): - tag = 'service_version' + tag = "service_version" def get_name(self): - return 'valid service version' + return "valid service version" def _is_valid(self, value): try: hyphen_index = len(value) for _ in range(3): - hyphen_index = value.rfind('-', 0, hyphen_index) - time = datetime.datetime.strptime(value[hyphen_index + 1:], '%Y-%m-%d') + hyphen_index = value.rfind("-", 0, hyphen_index) + time = datetime.datetime.strptime(value[hyphen_index + 1 :], "%Y-%m-%d") isdate = isinstance(time, datetime.date) except ValueError: isdate = False @@ -46,12 +49,13 @@ def _is_valid(self, value): class SourceKey(Validator): - """ Validate that curated source keys appear in curated/sources.yaml. """ - tag = 'source_key' - curated_sources = {} + """Validate that curated source keys appear in curated/sources.yaml.""" + + tag = "source_key" + curated_sources: dict[str, any] = {} def get_name(self): - return 'source key found in curated/sources.yaml' + return "source key found in curated/sources.yaml" def _is_valid(self, value): return value in self.curated_sources @@ -62,24 +66,26 @@ class ExampleId(Validator): Validate an example ID starts with a service ID and has underscore-separated operation and specializations (like sns_Subscribe_Email). """ - tag = 'example_id' - services = {} + + tag = "example_id" + services: dict[str, any] = {} def get_name(self): return "valid example ID" def _is_valid(self, value): - if not re.fullmatch('^[\\da-z-]+(_[\\da-zA-Z]+)+$', value): + if not re.fullmatch("^[\\da-z-]+(_[\\da-zA-Z]+)+$", value): return False else: - svc = value.split('_')[0] - return svc == 'cross' or svc in self.services + svc = value.split("_")[0] + return svc == "cross" or svc in self.services class BlockContent(Validator): - """ Validate that block content refers to an existing file. """ - tag = 'block_content' - block_names = [] + """Validate that block content refers to an existing file.""" + + tag = "block_content" + block_names: list[str] = [] def get_name(self): return "file found in the cross-content folder" @@ -89,61 +95,64 @@ def _is_valid(self, value): class StringExtension(String): - """ Validate that strings don't contain non-entity AWS usage. """ + """Validate that strings don't contain non-entity AWS usage.""" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.check_aws = bool(kwargs.pop('check_aws', True)) - self.upper_start = bool(kwargs.pop('upper_start', False)) - self.lower_start = bool(kwargs.pop('lower_start', False)) - self.end_punc = bool(kwargs.pop('end_punc', False)) - self.no_end_punc = bool(kwargs.pop('no_end_punc', False)) - self.end_punc_or_colon = bool(kwargs.pop('end_punc_or_colon', False)) - self.end_punc_or_semicolon = bool(kwargs.pop('end_punc_or_semicolon', False)) - self.last_err = 'valid string' + self.check_aws = bool(kwargs.pop("check_aws", True)) + self.upper_start = bool(kwargs.pop("upper_start", False)) + self.lower_start = bool(kwargs.pop("lower_start", False)) + self.end_punc = bool(kwargs.pop("end_punc", False)) + self.no_end_punc = bool(kwargs.pop("no_end_punc", False)) + self.end_punc_or_colon = bool(kwargs.pop("end_punc_or_colon", False)) + self.end_punc_or_semicolon = bool(kwargs.pop("end_punc_or_semicolon", False)) + self.last_err = "valid string" def get_name(self): return self.last_err def _is_valid(self, value): - if value == '': + if value == "": return True valid = True if self.check_aws: # All occurrences of AWS must be entities or within a word. - valid = len(re.findall('(? /dev/null && echo "yes") +HAS_3_12=$(python3.12 --version > /dev/null && echo "yes") + +if [ "yes" ~= "${HAS_3_11}${HAS_3_12}"] ; then +else + echo "No python3.11 or python3.12" + exit 1 +fi + +PYTHON3_MINOR=$(python3 --version | cut -d '.' -f 2) +if [ "$PYTHON3_MINOR" -lt 11 ] ; then + echo "python3 default minor version less than 11" + exit 1 +fi + +python3 -m pip install --upgrade pip +python3 -m pip install -r base_requirements.txt \ No newline at end of file