feat: add generate_repo.py (#2431)

In this PR:
- Add `generate_repo.py` to generate libraries defined in a
configuration file.
  - Regenerate `.repo-metadata.json`
  - Generate `.OwlBot.yaml` and `owlbot.py` for new clients
- Invoke `generate_composed_library.py` to generate multiple versions of
a GAPIC library.
  - Apply repo-level post processing using python
    - Regenerate root `pom.xml`
    - Regenerate `gapic-libraries-bom/pom.xml`
- Format python script using `black`.
- Add unit tests for utility functions.
- Add an integration test using python, replacing shell integration
tests.
- Add a python lint check in CI.

This is [step
2](https://docs.google.com/document/d/1JiCcG3X7lnxaJErKe0ES_JkyU7ECb40nf2Xez3gWvuo/edit?pli=1&tab=t.0#bookmark=id.h3n0hcp3ch2m)
of milestone 2 of hermetic build project.

.github/workflows/verify_library_generation.yaml

@@ -13,9 +13,7 @@ jobs:
     strategy:
       matrix:
         java: [ 11 ]
-        os: [ ubuntu-22.04, macos-12 ]
-        post_processing: [ 'true', 'false' ]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-java@v3
@@ -45,35 +43,13 @@ jobs:
         pushd library_generation
         pip install -r requirements.in
         popd
-
-    - name: install utils (macos)
-      if: matrix.os == 'macos-12'
-      shell: bash
-      run: |
-        brew update --preinstall
-        # we need the `realpath` command to be available
-        brew install coreutils
-    - name: install docker (ubuntu)
-      if: matrix.os == 'ubuntu-22.04'
-      shell: bash
-      run: |
-        set -x
-        # install docker
-        sudo apt install containerd -y
-        sudo apt install -y docker.io docker-compose
-
-        # launch docker
-        sudo systemctl start docker
     - name: Run integration tests
-      # we don't run ITs with postprocessing on macos because one of its dependencies "synthtool" is designed to run on linux only
-      if: matrix.os == 'ubuntu-22.04' || matrix.post_processing == 'false'
       shell: bash
       run: |
+        set -x
         git config --global user.email "[email protected]"
         git config --global user.name "Github Workflow"
-        library_generation/test/generate_library_integration_test.sh \
-        --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git \
-        --enable_postprocessing "${{ matrix.post_processing }}"
+        python -m unittest library_generation/test/integration_tests.py
   unit_tests:
     strategy:
       matrix:
@@ -106,7 +82,7 @@ jobs:
       run: |
         set -x
         python -m unittest library_generation/test/unit_tests.py
-  lint:
+  lint-shell:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3
@@ -116,3 +92,20 @@ jobs:
         scandir: 'library_generation'
         format: tty
         severity: error
+  lint-python:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: install python dependencies
+      shell: bash
+      run: |
+        set -ex
+        pushd library_generation
+        pip install -r requirements.in
+        popd
+    - name: Lint
+      shell: bash
+      run: |
+        # exclude generated golden files
+        # exclude owlbot until further refaction
+        black --check library_generation --exclude "(library_generation/owlbot)|(library_generation/test/resources/goldens)"

.gitignore

@@ -19,8 +19,12 @@ target/
 
 # Python
 **/__pycache__/
+.venv
 
 # library generation
 output/
 library_generation/output/
+library_generation/test/output
+library_generation/test/googleapis
+library_generation/test/resources/integration/golden
 showcase/scripts/output/

library_generation/generate_composed_library.py

@@ -1,3 +1,18 @@
+#!/usr/bin/env python3
+#  Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """
 This script allows generation of libraries that are composed of more than one
 service version. It is achieved by calling `generate_library.sh` without
@@ -12,130 +27,146 @@
   - A "grafeas" folder found in the googleapis/googleapis repository
 Note: googleapis repo is found in https://github.com/googleapis/googleapis.
 """
-
-import click
-import utilities as util
 import os
-import sys
-import subprocess
-import json
-from model.GenerationConfig import GenerationConfig
-from model.LibraryConfig import LibraryConfig
-from model.ClientInputs import parse as parse_build_file
+from pathlib import Path
+from typing import List
+import library_generation.utilities as util
+from library_generation.model.generation_config import GenerationConfig
+from library_generation.model.gapic_config import GapicConfig
+from library_generation.model.gapic_inputs import GapicInputs
+from library_generation.model.library_config import LibraryConfig
+from library_generation.model.gapic_inputs import parse as parse_build_file
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 
-"""
-Main function in charge of generating libraries composed of more than one
-service or service version.
-Arguments
- - config: a GenerationConfig object representing a parsed configuration
- yaml
- - library: a LibraryConfig object contained inside config, passed here for
-   convenience and to prevent all libraries to be processed
- - enable_postprocessing: true if postprocessing should be done on the generated
-   libraries
- - repository_path: path to the repository where the generated files will be
-   sent. If not specified, it will default to the one defined in the configuration yaml
-   and will be downloaded. The versions file will be inferred from this folder
-"""
+
 def generate_composed_library(
     config: GenerationConfig,
+    library_path: str,
     library: LibraryConfig,
-    repository_path: str,
-    enable_postprocessing: bool = True,
+    output_folder: str,
+    versions_file: str,
 ) -> None:
-  output_folder = util.sh_util('get_output_folder')
-
-  print(f'output_folder: {output_folder}')
-  print('library: ', library)
-  os.makedirs(output_folder, exist_ok=True)
-
-  googleapis_commitish = config.googleapis_commitish
-  if library.googleapis_commitish is not None:
-    googleapis_commitish = library.googleapis_commitish
-    print('using library-specific googleapis commitish: ' + googleapis_commitish)
-  else:
-    print('using common googleapis_commitish')
-
-  print('removing old googleapis folders and files')
-  util.delete_if_exists(f'{output_folder}/google')
-  util.delete_if_exists(f'{output_folder}/grafeas')
-
-  print('downloading googleapis')
-  util.sh_util(f'download_googleapis_files_and_folders "{output_folder}" "{googleapis_commitish}"')
-
-  is_monorepo = len(config.libraries) > 1
+    """
+    Generate libraries composed of more than one service or service version
+    :param config: a GenerationConfig object representing a parsed configuration
+    yaml
+    :param library_path: the path to which the generated file goes
+    :param library: a LibraryConfig object contained inside config, passed here
+    for convenience and to prevent all libraries to be processed
+    :param output_folder:
+    :param versions_file:
+    :return None
+    """
+    util.pull_api_definition(
+        config=config, library=library, output_folder=output_folder
+    )
+
+    is_monorepo = util.check_monorepo(config=config)
+    base_arguments = __construct_tooling_arg(config=config)
+    owlbot_cli_source_folder = util.sh_util("mktemp -d")
+    os.makedirs(f"{library_path}", exist_ok=True)
+    for gapic in library.gapic_configs:
+        build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve()
+        print(f"build_file_folder: {build_file_folder}")
+        gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path)
+        # generate prerequisite files (.repo-metadata.json, .OwlBot.yaml,
+        # owlbot.py) here because transport is parsed from BUILD.bazel,
+        # which lives in a versioned proto_path.
+        util.generate_prerequisite_files(
+            library=library,
+            proto_path=util.remove_version_from(gapic.proto_path),
+            transport=gapic_inputs.transport,
+            library_path=library_path,
+        )
+        service_version = gapic.proto_path.split("/")[-1]
+        temp_destination_path = f"java-{library.api_shortname}-{service_version}"
+        effective_arguments = __construct_effective_arg(
+            base_arguments=base_arguments,
+            gapic=gapic,
+            gapic_inputs=gapic_inputs,
+            temp_destination_path=temp_destination_path,
+        )
+        print("arguments: ")
+        print(effective_arguments)
+        print(f"Generating library from {gapic.proto_path} to {library_path}")
+        util.run_process_and_print_output(
+            ["bash", f"{script_dir}/generate_library.sh", *effective_arguments],
+            "Library generation",
+        )
+
+        util.sh_util(
+            f'build_owlbot_cli_source_folder "{library_path}"'
+            + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"'
+            + f' "{gapic.proto_path}"',
+            cwd=output_folder,
+        )
 
-  base_arguments = []
-  base_arguments += util.create_argument('gapic_generator_version', config)
-  base_arguments += util.create_argument('grpc_version', config)
-  base_arguments += util.create_argument('protobuf_version', config)
-
-  library_name = f'java-{library.api_shortname}'
-  library_path = None
-
-  versions_file = ''
-  if is_monorepo:
-    print('this is a monorepo library')
-    destination_path = config.destination_path + '/' + library_name
-    library_folder = destination_path.split('/')[-1]
-    if repository_path is None:
-      print(f'sparse_cloning monorepo with {library_name}')
-      repository_path = f'{output_folder}/{config.destination_path}'
-      clone_out = util.sh_util(f'sparse_clone "https://github.com/googleapis/{MONOREPO_NAME}.git" "{library_folder} google-cloud-pom-parent google-cloud-jar-parent versions.txt .github"', cwd=output_folder)
-      print(clone_out)
-    library_path = f'{repository_path}/{library_name}'
-    versions_file = f'{repository_path}/versions.txt'
-  else:
-    print('this is a HW library')
-    destination_path = library_name
-    if repository_path is None:
-      repository_path = f'{output_folder}/{destination_path}'
-      util.delete_if_exists(f'{output_folder}/{destination_path}')
-      clone_out = util.sh_util(f'git clone "https://github.com/googleapis/{destination_path}.git"', cwd=output_folder)
-      print(clone_out)
-    library_path = f'{repository_path}'
-    versions_file = f'{repository_path}/versions.txt'
-
-  owlbot_cli_source_folder = util.sh_util('mktemp -d')
-  for gapic in library.gapic_configs:
-
-    effective_arguments = list(base_arguments)
-    effective_arguments += util.create_argument('proto_path', gapic)
-
-    build_file_folder = f'{output_folder}/{gapic.proto_path}'
-    print(f'build_file_folder: {build_file_folder}')
-    client_inputs = parse_build_file(build_file_folder, gapic.proto_path)
-    effective_arguments += [
-        '--proto_only', client_inputs.proto_only,
-        '--gapic_additional_protos', client_inputs.additional_protos,
-        '--transport', client_inputs.transport,
-        '--rest_numeric_enums', client_inputs.rest_numeric_enum,
-        '--gapic_yaml', client_inputs.gapic_yaml,
-        '--service_config', client_inputs.service_config,
-        '--service_yaml', client_inputs.service_yaml,
-        '--include_samples', client_inputs.include_samples,
-    ]
-    service_version = gapic.proto_path.split('/')[-1]
-    temp_destination_path = f'java-{library.api_shortname}-{service_version}'
-    effective_arguments += [ '--destination_path', temp_destination_path ]
-    print('arguments: ')
-    print(effective_arguments)
-    print(f'Generating library from {gapic.proto_path} to {destination_path}...')
-    util.run_process_and_print_output(['bash', '-x', f'{script_dir}/generate_library.sh',
-      *effective_arguments], 'Library generation')
-
-
-    if enable_postprocessing:
-      util.sh_util(f'build_owlbot_cli_source_folder "{library_path}"'
-                   + f' "{owlbot_cli_source_folder}" "{output_folder}/{temp_destination_path}"'
-                   + f' "{gapic.proto_path}"',
-                   cwd=output_folder)
-
-  if enable_postprocessing:
     # call postprocess library
-    util.run_process_and_print_output([f'{script_dir}/postprocess_library.sh',
-              f'{library_path}', '', versions_file, owlbot_cli_source_folder,
-                                       config.owlbot_cli_image, config.synthtool_commitish, str(is_monorepo).lower()], 'Library postprocessing')
+    util.run_process_and_print_output(
+        [
+            f"{script_dir}/postprocess_library.sh",
+            f"{library_path}",
+            "",
+            versions_file,
+            owlbot_cli_source_folder,
+            config.owlbot_cli_image,
+            config.synthtool_commitish,
+            str(is_monorepo).lower(),
+        ],
+        "Library postprocessing",
+    )
+
+
+def __construct_tooling_arg(config: GenerationConfig) -> List[str]:
+    """
+    Construct arguments of tooling versions used in generate_library.sh
+    :param config: the generation config
+    :return: arguments containing tooling versions
+    """
+    arguments = []
+    arguments += util.create_argument("gapic_generator_version", config)
+    arguments += util.create_argument("grpc_version", config)
+    arguments += util.create_argument("protobuf_version", config)
+
+    return arguments
+
+
+def __construct_effective_arg(
+    base_arguments: List[str],
+    gapic: GapicConfig,
+    gapic_inputs: GapicInputs,
+    temp_destination_path: str,
+) -> List[str]:
+    """
+    Construct arguments consist attributes of a GAPIC library which used in
+    generate_library.sh
+    :param base_arguments: arguments consist of tooling versions
+    :param gapic: an object of GapicConfig
+    :param gapic_inputs: an object of GapicInput
+    :param temp_destination_path: the path to which the generated library goes
+    :return: arguments containing attributes to generate a GAPIC library
+    """
+    arguments = list(base_arguments)
+    arguments += util.create_argument("proto_path", gapic)
+    arguments += [
+        "--proto_only",
+        gapic_inputs.proto_only,
+        "--gapic_additional_protos",
+        gapic_inputs.additional_protos,
+        "--transport",
+        gapic_inputs.transport,
+        "--rest_numeric_enums",
+        gapic_inputs.rest_numeric_enum,
+        "--gapic_yaml",
+        gapic_inputs.gapic_yaml,
+        "--service_config",
+        gapic_inputs.service_config,
+        "--service_yaml",
+        gapic_inputs.service_yaml,
+        "--include_samples",
+        gapic_inputs.include_samples,
+    ]
+    arguments += ["--destination_path", temp_destination_path]
 
+    return arguments