feat: move synthtool templates to library_generation/owlbot

.github/workflows/verify_library_generation.yaml

@@ -98,6 +98,21 @@ jobs:
         pushd library_generation
         pip install -r requirements.in
         popd
+    - name: install synthtool
+      shell: bash
+      run: |
+        set -ex
+        mkdir -p /tmp/synthtool
+        pushd /tmp/synthtool
+        if [ ! -d "synthtool" ]; then
+          git clone https://github.com/googleapis/synthtool.git
+        fi
+        pushd "synthtool"
+
+        git reset --hard origin/no-java-templates
+
+        python3 -m pip install -e .
+        python3 -m pip install -r requirements.in
     - name: Run shell unit tests
       run: |
         set -x

library_generation/generate_composed_library.py

@@ -137,5 +137,5 @@ def generate_composed_library(
     # call postprocess library
     util.run_process_and_print_output([f'{script_dir}/postprocess_library.sh',
               f'{library_path}', '', versions_file, owlbot_cli_source_folder,
-                                       config.owlbot_cli_image, config.synthtool_commitish, str(is_monorepo).lower()], 'Library postprocessing')
+                                       config.owlbot_cli_image, config.synthtool_commitish, str(is_monorepo).lower(), config.path_to_yaml], 'Library postprocessing')
 

library_generation/model/GenerationConfig.py

@@ -17,6 +17,8 @@ def __init__(
       owlbot_cli_image: str,
       synthtool_commitish: str,
       destination_path: Optional[str],
+      template_excludes: str,
+      path_to_yaml: str,
       libraries: List[LibraryConfig],
   ):
     self.gapic_generator_version = gapic_generator_version
@@ -26,6 +28,8 @@ def __init__(
     self.owlbot_cli_image = owlbot_cli_image
     self.synthtool_commitish = synthtool_commitish
     self.destination_path = destination_path
+    self.template_excludes = template_excludes
+    self.path_to_yaml = path_to_yaml
     self.libraries = libraries
 
   """
@@ -75,6 +79,8 @@ def from_yaml(path_to_yaml: str):
       _required(config, 'owlbot_cli_image'),
       _required(config, 'synthtool_commitish'),
       _optional(config, 'destination_path', None),
+      _required(config, 'template_excludes'),
+      path_to_yaml,
       parsed_libraries
     )
 

library_generation/owlbot/bin/entrypoint.sh

@@ -26,50 +26,20 @@
 set -ex
 scripts_root=$1
 versions_file=$2
+configuration_yaml=$3
 
-# Runs template and etc in current working directory
-function processModule() {
-  # templates as well as retrieving files from owl-bot-staging
-  echo "Generating templates and retrieving files from owl-bot-staging directory..."
-  if [ -f "owlbot.py" ]
-  then
-    # defaults to run owlbot.py
-    python3 owlbot.py
-  fi
-  echo "...done"
-
-  # write or restore pom.xml files
-  echo "Generating missing pom.xml..."
-  python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
-  echo "...done"
-
-  # write or restore clirr-ignored-differences.xml
-  echo "Generating clirr-ignored-differences.xml..."
-  ${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
-  echo "...done"
-
-  # fix license headers
-  echo "Fixing missing license headers..."
-  python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
-  echo "...done"
-
-  # TODO: re-enable this once we resolve thrashing
-  # restore license headers years
-  # echo "Restoring copyright years..."
-  # /owlbot/bin/restore_license_headers.sh
-  # echo "...done"
-
-  # ensure formatting on all .java files in the repository
-  echo "Reformatting source..."
-  mvn fmt:format
-  echo "...done"
-}
+pushd "${scripts_root}"
+# install library_generation which contains model classes used by the the
+# template rendering step
+python -m pip install -e .
+popd #scripts_root
 
 # This script can be used to process HW libraries and monorepo
 # (google-cloud-java) libraries, which require a slightly different treatment
 # monorepo folders have an .OwlBot.yaml file in the module folder (e.g.
 # java-asset/.OwlBot.yaml), whereas HW libraries have the yaml in
 # `.github/.OwlBot.yaml`
+monorepo="false"
 if [[ -f "$(pwd)/.OwlBot.yaml" ]]; then
   monorepo="true"
 fi
@@ -80,4 +50,48 @@ if [[ "${monorepo}" == "true" ]]; then
   mv temp owl-bot-staging
 fi
 
-processModule
+
+# Runs template and etc in current working directory
+monorepo=$1
+
+# apply repo templates
+echo "Rendering templates"
+python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}"
+
+# templates as well as retrieving files from owl-bot-staging
+echo "Retrieving files from owl-bot-staging directory..."
+if [ -f "owlbot.py" ]
+then
+  # defaults to run owlbot.py
+  python3 owlbot.py
+fi
+echo "...done"
+
+# write or restore pom.xml files
+echo "Generating missing pom.xml..."
+python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
+echo "...done"
+
+# write or restore clirr-ignored-differences.xml
+echo "Generating clirr-ignored-differences.xml..."
+${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
+echo "...done"
+
+# fix license headers
+echo "Fixing missing license headers..."
+python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
+echo "...done"
+
+# TODO: re-enable this once we resolve thrashing
+# restore license headers years
+# echo "Restoring copyright years..."
+# /owlbot/bin/restore_license_headers.sh
+# echo "...done"
+
+# ensure formatting on all .java files in the repository
+echo "Reformatting source..."
+mvn fmt:format
+echo "...done"
+
+
+

library_generation/owlbot/src/apply_repo_templates.py

@@ -0,0 +1,43 @@
+"""
+This script parses an owlbot.py file, specifically the call to `java.common_templates` in
+order to extract the excluded files so it can be called with a custom template path
+pointing to the templates hosted in `sdk-platform-java/library_generation/owlbot/templates`.
+Briefly, this wraps the call to synthtool's common templates using a custom template folder.
+"""
+
+import os
+import sys
+from collections.abc import Sequence
+from synthtool.languages.java import common_templates
+from pathlib import Path
+from model.GenerationConfig import GenerationConfig
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+repo_templates_path = os.path.join(script_dir, '..', 'templates', 'java_library')
+
+def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None:
+  config = GenerationConfig.from_yaml(configuration_yaml_path)
+  print(f'repo_templates_path: {repo_templates_path}')
+  print(f'excludes: {config.template_excludes}')
+  common_templates(
+      excludes=config.template_excludes,
+      template_path=Path(repo_templates_path),
+      monorepo=monorepo
+  )
+
+
+def main(argv: Sequence[str]) -> None:
+  if len(argv) != 3:
+    raise ValueError("Usage: python apply-repo-templates.py configuration_yaml_path monorepo")
+
+  configuration_yaml_path = argv[1]
+  monorepo = argv[2]
+  apply_repo_templates(
+      configuration_yaml_path,
+      monorepo.lower() == 'true'
+  )
+
+
+
+if __name__ == "__main__":
+  main(sys.argv)

library_generation/owlbot/src/requirements.in

@@ -8,4 +8,3 @@ colorlog
 protobuf
 watchdog
 requests
-pyyaml

library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md

@@ -0,0 +1,94 @@
+<!-- # Generated by synthtool. DO NOT EDIT! !-->
+# Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of
+experience, education, socio-economic status, nationality, personal appearance,
+race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+*   Using welcoming and inclusive language
+*   Being respectful of differing viewpoints and experiences
+*   Gracefully accepting constructive criticism
+*   Focusing on what is best for the community
+*   Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+*   The use of sexualized language or imagery and unwelcome sexual attention or
+    advances
+*   Trolling, insulting/derogatory comments, and personal or political attacks
+*   Public or private harassment
+*   Publishing others' private information, such as a physical or electronic
+    address, without explicit permission
+*   Other conduct which could reasonably be considered inappropriate in a
+    professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, or to ban temporarily or permanently any
+contributor for other behaviors that they deem inappropriate, threatening,
+offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+This Code of Conduct also applies outside the project spaces when the Project
+Steward has a reasonable belief that an individual's behavior may have a
+negative impact on the project or its community.
+
+## Conflict Resolution
+
+We do not believe that all conflict is bad; healthy debate and disagreement
+often yield positive results. However, it is never okay to be disrespectful or
+to engage in behavior that violates the project’s code of conduct.
+
+If you see someone violating the code of conduct, you are encouraged to address
+the behavior directly with those involved. Many issues can be resolved quickly
+and easily, and this gives people more control over the outcome of their
+dispute. If you are unable to resolve the matter for any reason, or if the
+behavior is threatening or harassing, report it. We are dedicated to providing
+an environment where participants feel welcome and safe.
+
+Reports should be directed to *[email protected]*, the
+Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to
+receive and address reported violations of the code of conduct. They will then
+work with a committee consisting of representatives from the Open Source
+Programs Office and the Google Open Source Strategy team. If for any reason you
+are uncomfortable reaching out to the Project Steward, please email
+[email protected].
+
+We will investigate every complaint, but you may not receive a direct response.
+We will use our discretion in determining when and how to follow up on reported
+incidents, which may range from not taking action to permanent expulsion from
+the project and project-sponsored spaces. We will notify the accused of the
+report and provide them an opportunity to discuss it before any action is taken.
+The identity of the reporter will be omitted from the details of the report
+supplied to the accused. In potentially harmful situations, such as ongoing
+harassment or threats to anyone's safety, we may take action without notice.
+
+## Attribution
+
+This Code of Conduct is adapted from the Contributor Covenant, version 1.4,
+available at
+https://www.contributor-covenant.org/version/1/4/code-of-conduct.html