Tools: Documented example schema for SOS guide (#5573)

* Documented example schema for SOS guide
* Normalized validation regex
* Streamline to_validate loop

.tools/validation/schema/curated_example_schema.yaml

@@ -1,7 +1,7 @@
 # Yamale Schema for curated example metadata, which is all .yaml files in the metadata/curated folder
 # with a _metadata.yaml suffix.
 
-map(include('curated_example'), key=regex('^[\da-z_-]+$', name='valid curated example ID'))
+map(include('curated_example'), key=regex('^[-_a-z0-9]+$', name='valid curated example ID'))
 ---
 curated_example:
   title: str(upper_start=True, no_end_punc=True)
@@ -22,4 +22,4 @@ language:
   description_list: list(str(upper_start=True, end_punc=True), required=False)
   source_url:
     text: str(no_end_punc=True)
-    href: regex('^(http).+', name="URL")
+    href: regex('^http', name="URL")

.tools/validation/schema/curated_sources_schema.yaml

@@ -1,8 +1,8 @@
 # Yamale Schema for curated sources metadata, which is the sources.yaml file in the metadata/curated folder.
 
-map(include('source'), key=regex('^[\da-z-]+$', name='source name'))
+map(include('source'), key=regex('^[-a-z0-9]+$', name='source name'))
 ---
 source:
   name: str(upper_start=True, no_end_punc=True)
   description: str(upper_start=True, end_punc=True)
-  url: regex('^(http).+', name="URL")
+  url: regex('^http', name="URL")

.tools/validation/schema/example_schema.yaml

@@ -3,36 +3,57 @@
 
 map(include('example'), key=example_id())
 ---
+# An example blocks all the languages together for a single example in a tab list. It is a navigable page on the code examples library. It is the top level "unit" of SoS content. This metadata is merged from tributaries with aws-doc-sdk-examples.
 example:
-  title: str(upper_start=True, no_end_punc=True)
-  title_abbrev: str(upper_start=True, no_end_punc=True)
-  synopsis: str(required=False, lower_start=True, end_punc_or_semicolon=True)
-  synopsis_list: list(str(upper_start=True, end_punc=True), required=False)
-  category: str(required=False, upper_start=True, no_end_punc=True)
-  guide_topic: include('guide_topic', required=False)
+  # Human readable title. TODO: Defaults to slug-to-title of the ID if not provided. Overwritten by aws-doc-sdk-example when merging.
+  title: str(upper_start=True, no_end_punc=True, required=False)
+  # Used in the TOC. TODO: Defaults to slug-to-title of the ID if not provided. Overwritten by aws-doc-sdk-example when merging.
+  title_abbrev: str(upper_start=True, no_end_punc=True, required=False)
+  # String label categories. Categories inferred by cross-service with multiple services, and can be whatever else it wants. Controls where in the TOC it appears. Overwritten by aws-doc-sdk-example when merging.
+  category: str(upper_start=True, no_end_punc=True, required=False)
+  # Link to additional topic places. Overwritten by aws-doc-sdk-example when merging.
+  guide_topic: include('guide_topic', required=False) # TODO Make this a list or a single.
+  # TODO how to add a language here and require it in sdks_schema. TODO: Keys merged by aws-doc-sdk-example when merging.
   languages: map(include('language'), key=enum('Bash', 'C++', 'CLI', 'Go', 'Java', 'JavaScript', 'Kotlin', '.NET', 'PHP', 'Python', 'Ruby', 'Rust', 'SAP ABAP', 'Swift'))
+  # TODO document service_main and services. Not to be used by tributaries. Part of Cross Service.
+  # List of services used by the examples. Lines up with those in services.yaml. Overwritten by aws-doc-sdk-example when merging.
   service_main: service_name(required=False)
   services: map(map(key=str(), required=False), key=service_name())
+  synopsis: str(required=False, lower_start=True, end_punc_or_semicolon=True, required=False)
+  synopsis_list: list(str(upper_start=True, end_punc=True), required=False)
 
+# Used for creating links in the block.
 guide_topic:
   title: str(upper_start=True, no_end_punc=True)
   url: include('doc_url', required=False)
 
+# Language Version configuration. Likely just the single list item.
 language:
   versions: list(include('version'))
 
+# Example for a single Language.
 version:
   sdk_version: int(min=1)
-  github: regex('^(?!http).+', name="relative URL", required=False)
-  sdkguide: include('doc_url', required=False)
-  excerpts: list(include('excerpt'), required=False)
+  # Additional ZonBook XML to include in the tab for this sample.
   block_content: block_content(required=False)
+  # The specific code samples to include in the example.
+  excerpts: list(include('excerpt'), required=False)
+  # Link to the source code for this example. TODO rename.
+  github: str(required=False)
   add_services: map(key=service_name(), required=False)
+  # Deprecated. Replace with guide_topic list.
+  sdkguide: include('doc_url', required=False)
+  # Link to additional topic places. TODO: Overwritten by aws-doc-sdk-example when merging.
+  more_info: list(include('guide_topic', required=False))
 
+# One language example can have several excerpts, each having a description block and one or more snippets.
+# An excerpt may have either snippet_files OR snippet_tags, but not both.
 excerpt:
   description: str(required=False, upper_start=True, end_punc=True)
-  snippet_tags: list(str(), required=False)
+  # A path within the repo to extract the entire file as a snippet.
   snippet_files: list(str(), required=False)
+  # Tags embedded in source files to extract as snippets.
+  snippet_tags: list(str(), required=False)
 
-service_slug_regex: regex('^[\da-z-]+$', name='service slug')
+service_slug_regex: regex('^[-a-z0-9]+$', name='service slug')
 doc_url: regex('^(?!https://docs.aws.amazon.com/).+', name="relative documentation URL")

.tools/validation/schema/example_strict_schema.yaml

@@ -0,0 +1,40 @@
+# Yamale Schema for example metadata, which is all .yaml files in the metadata folder
+# with a _metadata.yaml suffix.
+
+map(include('example'), key=example_id())
+---
+example:
+  title: str(upper_start=True, no_end_punc=True)
+  title_abbrev: str(upper_start=True, no_end_punc=True)
+  synopsis: str(required=False, lower_start=True, end_punc_or_semicolon=True)
+  synopsis_list: list(str(upper_start=True, end_punc=True), required=False)
+  category: str(required=False, upper_start=True, no_end_punc=True)
+  guide_topic: include('guide_topic', required=False)
+  languages: map(include('language'), key=enum('Bash', 'C++', 'CLI', 'Go', 'Java', 'JavaScript', 'Kotlin', '.NET', 'PHP', 'Python', 'Ruby', 'Rust', 'SAP ABAP', 'Swift'))
+  service_main: service_name(required=False)
+  services: map(map(key=str(), required=False), key=service_name())
+
+guide_topic:
+  title: str(upper_start=True, no_end_punc=True)
+  url: include('doc_url', required=False)
+
+language:
+  versions: list(include('version'))
+
+# Per-language excerpts for the example. Languages and SDK versions are defined in .doc_gen/metadata/sdk_metadata.yaml
+version:
+  sdk_version: int(min=1)
+  github: regex('^(?!http).+', name="relative URL", required=False)
+  sdkguide: include('doc_url', required=False)
+  excerpts: list(include('excerpt'), required=False)
+  block_content: block_content(required=False)
+  add_services: map(key=service_name(), required=False)
+
+# The references to code content that will be included in the example's content.
+excerpt:
+  description: str(required=False, upper_start=True, end_punc=True)
+  snippet_tags: list(str(), required=False)
+  snippet_files: list(str(), required=False)
+
+service_slug_regex: regex('^[-a-z0-9]+$', name='service slug')
+doc_url: regex('^(?!https://docs.aws.amazon.com/).+', name="relative documentation URL")

.tools/validation/schema/sdks_schema.yaml

@@ -29,5 +29,5 @@ title_override:
   title_abbrev: str()
 
 syntax_enum: enum('bash', 'cpp', 'go', 'java', 'javascript', 'kotlin', 'csharp', 'php', 'python', 'ruby', 'rust', 'sap-abap', 'sh', 'swift')
-entity_regex: regex('^[&]([\dA-Za-z-_])+[;]$', name='valid entity')
-entity_with_version_regex: regex('^[&]([\dA-Za-z-_])+;', name='valid entity with version')
+entity_regex: regex('^&[-_a-zA-Z0-9]+;$', name='valid entity')
+entity_with_version_regex: regex('^&[-_a-zA-Z0-9]+;', name='valid entity with version')

.tools/validation/schema/services_schema.yaml

@@ -1,6 +1,6 @@
 # Yamale Schema for services metadata, which is the services.yaml file in the metadata folder.
 
-map(include('service'), key=regex('^[\da-z-]+$', name='service slug'))
+map(include('service'), key=regex('^[-a-z0-9]+$', name='service slug'))
 ---
 service:
   long: include('long_entity_regex')
@@ -22,9 +22,9 @@ service:
   tags: map(key=enum('product_categories'))
 
 chapter_override:
-    title: str(end_punc=False)
-    title_abbrev: str(end_punc=False)
+  title: str(end_punc=False)
+  title_abbrev: str(end_punc=False)
 
-long_entity_regex: regex('^[&]([\dA-Za-z-_])+[;]( \(&([\dA-Za-z-_])+;\))?$', name='valid entity')
-entity_regex: regex('^[&]([\dA-Za-z-_])+[;]$', name='valid entity')
-doc_url: regex('^(?!https://docs.aws.amazon.com/).+', name="relative documentation URL")
+long_entity_regex: regex('^&[-_a-zA-Z0-9]+;( \(&[-_a-zA-Z0-9]+;\))?$', name='valid entity')
+entity_regex: regex('^&[-_a-zA-Z0-9]+;$', name='valid entity')
+doc_url: regex('^(?!https://docs.aws.amazon.com/).+', name="relative documentation URL")

.tools/validation/validate_doc_metadata.py

@@ -7,12 +7,12 @@
 
 import argparse
 import datetime
-import glob
 import os
 import re
 import yaml
 import yamale
 from pathlib import Path
+from typing import Iterable
 from yamale import YamaleError
 from yamale.validators import DefaultValidators, Validator, String
 
@@ -151,7 +151,7 @@ def _is_valid(self, value):
         return valid
 
 
-def validate_files(schema_name, meta_names, validators):
+def validate_files(schema_name: Path, meta_names: Iterable[Path], validators):
     """Iterate a list of files and validate each one against a schema."""
     success = True
 
@@ -160,30 +160,31 @@ def validate_files(schema_name, meta_names, validators):
         try:
             data = yamale.make_data(meta_name)
             yamale.validate(schema, data)
-            print(f"{meta_name} validation success! 👍")
+            print(f"{meta_name.resolve()} validation success! 👍")
         except YamaleError as e:
             print(e.message)
             success = False
     return success
 
 
 def validate_all(doc_gen: Path):
-    # with open(os.path.join(args.doc_gen, "metadata/sdks.yaml")) as sdks_file:
+    # with open(doc_gen / "metadata" / "sdks.yaml") as sdks_file:
     #     sdks_yaml: dict[str, any] = yaml.safe_load(sdks_file)
 
-    with open(os.path.join(doc_gen, "metadata/services.yaml")) as services_file:
+    with open(doc_gen / "metadata" / "services.yaml") as services_file:
         services_yaml = yaml.safe_load(services_file)
 
     with open(
-        os.path.join(doc_gen, "metadata/curated/sources.yaml")
+        doc_gen / "metadata" / "curated" / "sources.yaml"
     ) as curated_sources_file:
         curated_sources_yaml = yaml.safe_load(curated_sources_file)
 
-    validators = DefaultValidators.copy()
     ServiceName.services = services_yaml
     SourceKey.curated_sources = curated_sources_yaml
     ExampleId.services = services_yaml
-    BlockContent.block_names = os.listdir(os.path.join(doc_gen, "cross-content"))
+    BlockContent.block_names = os.listdir(doc_gen / "cross-content")
+
+    validators = DefaultValidators.copy()
     validators[ServiceName.tag] = ServiceName
     validators[ServiceVersion.tag] = ServiceVersion
     validators[SourceKey.tag] = SourceKey
@@ -193,30 +194,20 @@ def validate_all(doc_gen: Path):
 
     schema_root = Path(__file__).parent / "schema"
 
-    # Validate sdks.yaml file.
-    schema_name = schema_root / "sdks_schema.yaml"
-    meta_names = glob.glob(os.path.join(doc_gen, "metadata/sdks.yaml"))
-    success = validate_files(schema_name, meta_names, validators)
-
-    # Validate services.yaml file.
-    schema_name = schema_root / "services_schema.yaml"
-    meta_names = glob.glob(os.path.join(doc_gen, "metadata/services.yaml"))
-    success &= validate_files(schema_name, meta_names, validators)
-
-    # Validate example (*_metadata.yaml in metadata folder) files.
-    schema_name = schema_root / "example_schema.yaml"
-    meta_names = glob.glob(os.path.join(doc_gen, "metadata/*_metadata.yaml"))
-    success &= validate_files(schema_name, meta_names, validators)
-
-    # Validate curated/sources.yaml file.
-    schema_name = schema_root / "curated_sources_schema.yaml"
-    meta_names = glob.glob(os.path.join(doc_gen, "metadata/curated/sources.yaml"))
-    success &= validate_files(schema_name, meta_names, validators)
-
-    # Validate curated example (*_metadata.yaml in metadata/curated folder) files.
-    schema_name = schema_root / "curated_example_schema.yaml"
-    meta_names = glob.glob(os.path.join(doc_gen, "metadata/curated/*_metadata.yaml"))
-    success &= validate_files(schema_name, meta_names, validators)
+    to_validate = [
+        # (schema, metadata_glob)
+        ("sdks_schema.yaml", "sdks.yaml"),
+        ("services_schema.yaml", "services.yaml"),
+        # TODO: Switch between strict schema for aws-doc-sdk-examples and loose schema for tributaries
+        ("example_strict_schema.yaml", "*_metadata.yaml"),
+        ("curated_sources_schema.yaml", "curated/sources.yaml"),
+        ("curated_example_schema.yaml", "curated/*_metadata.yaml"),
+    ]
+    success = True
+    for schema, metadata in to_validate:
+        success &= validate_files(
+            schema_root / schema, (doc_gen / "metadata").glob(metadata), validators
+        )
 
     return success
 
@@ -231,7 +222,7 @@ def main():
     )
     args = parser.parse_args()
 
-    success = validate_all(args.doc_gen)
+    success = validate_all(Path(args.doc_gen))
 
     if success:
         print("Validation succeeded! 👍👍👍")