From ba38e9c9a188b02d0899514d277b0c1618975536 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 2 Feb 2024 20:34:20 +0000 Subject: [PATCH 01/30] transfer templates from synthtool to owlbot --- library_generation/owlbot/bin/entrypoint.sh | 7 + .../owlbot/src/apply-repo-templates.py | 38 +++ library_generation/owlbot/src/requirements.in | 3 +- .../templates/java_library/CODE_OF_CONDUCT.md | 94 ++++++ .../templates/java_library/CONTRIBUTING.md | 92 ++++++ .../owlbot/templates/java_library/LICENSE | 201 ++++++++++++ .../owlbot/templates/java_library/README.md | 288 ++++++++++++++++++ .../owlbot/templates/java_library/SECURITY.md | 7 + .../owlbot/templates/java_library/java.header | 15 + .../templates/java_library/license-checks.xml | 10 + .../templates/java_library/renovate.json | 80 +++++ .../samples/install-without-bom/pom.xml | 86 ++++++ .../templates/java_library/samples/pom.xml | 56 ++++ .../java_library/samples/snapshot/pom.xml | 85 ++++++ .../java_library/samples/snippets/pom.xml | 49 +++ 15 files changed, 1110 insertions(+), 1 deletion(-) create mode 100644 library_generation/owlbot/src/apply-repo-templates.py create mode 100644 library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md create mode 100644 library_generation/owlbot/templates/java_library/CONTRIBUTING.md create mode 100644 library_generation/owlbot/templates/java_library/LICENSE create mode 100644 library_generation/owlbot/templates/java_library/README.md create mode 100644 library_generation/owlbot/templates/java_library/SECURITY.md create mode 100644 library_generation/owlbot/templates/java_library/java.header create mode 100644 library_generation/owlbot/templates/java_library/license-checks.xml create mode 100644 library_generation/owlbot/templates/java_library/renovate.json create mode 100644 library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml create mode 100644 library_generation/owlbot/templates/java_library/samples/pom.xml create mode 100644 library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml create mode 100644 library_generation/owlbot/templates/java_library/samples/snippets/pom.xml diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 26ed707591..80c4037689 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -29,6 +29,13 @@ versions_file=$2 # Runs template and etc in current working directory function processModule() { + # apply repo templates + if [ -f "owlbot.py" ] + then + # defaults to run owlbot.py + python3 "${scripts_root}/owlbot/src/apply-repo-templates.py" owlbot.py + fi + # templates as well as retrieving files from owl-bot-staging echo "Generating templates and retrieving files from owl-bot-staging directory..." if [ -f "owlbot.py" ] diff --git a/library_generation/owlbot/src/apply-repo-templates.py b/library_generation/owlbot/src/apply-repo-templates.py new file mode 100644 index 0000000000..215e0286b7 --- /dev/null +++ b/library_generation/owlbot/src/apply-repo-templates.py @@ -0,0 +1,38 @@ +"""TODO(diegomarquezp): DO NOT SUBMIT without one-line documentation for apply-repo-templates. + +TODO(diegomarquezp): DO NOT SUBMIT without a detailed description of apply-repo-templates. +""" + +import os +from collections.abc import Sequence +from absl import app +from synthtool.languages.java import common_templates +from pathlib import Path + +script_dir = os.path.dirname(os.path.realpath(__file__)) +repo_templates_path = os.path.join(script_dir, '..', 'templates', 'java_library') + +def apply_repo_templates(owlbot_py_path: str) -> None: + excludes = [] + with open(owlbot_py_path) as contents: + excludes += _parse_template_excludes(contents) + print(f'repo_templates_path: {repo_templates_path}') + print(f'excludes: {excludes}') + common_templates(excludes=excludes, template_path=Path(repo_templates_path)) + + +def _parse_template_excludes(owlbot_py_contents: str) -> str: + return [ 'README.md' ] + pass + +def main(argv: Sequence[str]) -> None: + if len(argv) != 2: + raise app.UsageError("Usage: python apply-repo-templates.py [owlbot_py_path]") + + owlbot_py_path = argv[1] + apply_repo_templates(owlbot_py_path) + + + +if __name__ == "__main__": + app.run(main) diff --git a/library_generation/owlbot/src/requirements.in b/library_generation/owlbot/src/requirements.in index 1dbbb3c666..a4b12718fa 100644 --- a/library_generation/owlbot/src/requirements.in +++ b/library_generation/owlbot/src/requirements.in @@ -8,4 +8,5 @@ colorlog protobuf watchdog requests -pyyaml \ No newline at end of file +pyyaml +absl-py diff --git a/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md b/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..2add2547a8 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/CODE_OF_CONDUCT.md @@ -0,0 +1,94 @@ + +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + +Reports should be directed to *googleapis-stewards@google.com*, the +Project Steward(s) for *Google Cloud Client Libraries*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out to the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html \ No newline at end of file diff --git a/library_generation/owlbot/templates/java_library/CONTRIBUTING.md b/library_generation/owlbot/templates/java_library/CONTRIBUTING.md new file mode 100644 index 0000000000..b65dd279c9 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/CONTRIBUTING.md @@ -0,0 +1,92 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). + +## Building the project + +To build, package, and run all unit tests run the command + +``` +mvn clean verify +``` + +### Running Integration tests + +To include integration tests when building the project, you need access to +a GCP Project with a valid service account. + +For instructions on how to generate a service account and corresponding +credentials JSON see: [Creating a Service Account][1]. + +Then run the following to build, package, run all unit tests and run all +integration tests. + +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account.json +mvn -Penable-integration-tests clean verify +``` + +## Code Samples + +All code samples must be in compliance with the [java sample formatting guide][3]. +Code Samples must be bundled in separate Maven modules. + +The samples must be separate from the primary project for a few reasons: +1. Primary projects have a minimum Java version of Java 8 whereas samples can have + Java version of Java 11. Due to this we need the ability to + selectively exclude samples from a build run. +2. Many code samples depend on external GCP services and need + credentials to access the service. +3. Code samples are not released as Maven artifacts and must be excluded from + release builds. + +### Building + +```bash +mvn clean verify +``` + +Some samples require access to GCP services and require a service account: + +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service/account.json +mvn clean verify +``` + +### Code Formatting + +Code in this repo is formatted with +[google-java-format](https://github.com/google/google-java-format). +To run formatting on your project, you can run: +``` +mvn com.coveo:fmt-maven-plugin:format +``` + +[1]: https://cloud.google.com/docs/authentication/getting-started#creating_a_service_account +[2]: https://maven.apache.org/settings.html#Active_Profiles +[3]: https://github.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md \ No newline at end of file diff --git a/library_generation/owlbot/templates/java_library/LICENSE b/library_generation/owlbot/templates/java_library/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/library_generation/owlbot/templates/java_library/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md new file mode 100644 index 0000000000..e849a97147 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/README.md @@ -0,0 +1,288 @@ +{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} +{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} +{% set repo_short = metadata['repo']['repo'].split('/')|last -%} + +# Google {{ metadata['repo']['name_pretty'] }} Client for Java + +Java idiomatic client for [{{metadata['repo']['name_pretty']}}][product-docs]. + +[![Maven][maven-version-image]][maven-version-link] +![Stability][stability-image] + +- [Product Documentation][product-docs] +- [Client Library Documentation][javadocs] +{% if 'partials' in metadata and metadata['partials']['deprecation_warning'] -%} +{{ metadata['partials']['deprecation_warning'] }} +{% elif metadata['repo']['release_level'] in ['preview'] %} +> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. +{% endif %} +{% if migrated_split_repo %} +:bus: In October 2022, this library has moved to +[google-cloud-java/{{ metadata['repo']['repo_short'] }}]( +https://github.com/googleapis/google-cloud-java/tree/main/{{ metadata['repo']['repo_short'] }}). +This repository will be archived in the future. +Future releases will appear in the new repository (https://github.com/googleapis/google-cloud-java/releases). +The Maven artifact coordinates (`{{ group_id }}:{{ artifact_id }}`) remain the same. +{% endif %} +## Quickstart + +{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] -%} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml +{{ metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] }} +``` + +If you are using Maven without the BOM, add this to your dependencies: +{% elif monorepo %} +If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: + +```xml + + + + com.google.cloud + libraries-bom + {{ metadata['latest_bom_version'] }} + pom + import + + + + + + + {{ group_id }} + {{ artifact_id }} + +``` + +If you are using Maven without the BOM, add this to your dependencies: +{% else %} +If you are using Maven, add this to your pom.xml file: +{% endif %} + + +```xml +{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_without_bom'] -%} +{{ metadata['snippets'][metadata['repo']['api_shortname'] + '_install_without_bom'] }} +{% else -%} + + {{ group_id }} + {{ artifact_id }} + {{ metadata['latest_version'] }} + +{% endif -%} +``` + +{% if 'snippets' in metadata and metadata['snippets'][metadata['repo']['api_shortname'] + '_install_with_bom'] -%} +If you are using Gradle 5.x or later, add this to your dependencies: + +```Groovy +implementation platform('com.google.cloud:libraries-bom:{{metadata['latest_bom_version']}}') + +implementation '{{ group_id }}:{{ artifact_id }}' +``` +{% endif -%} + +If you are using Gradle without BOM, add this to your dependencies: + +```Groovy +implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['latest_version'] }}' +``` + +If you are using SBT, add this to your dependencies: + +```Scala +libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['latest_version'] }}" +``` + + +## Authentication + +See the [Authentication][authentication] section in the base directory's README. + +## Authorization + +The client application making API calls must be granted [authorization scopes][auth-scopes] required for the desired {{metadata['repo']['name_pretty']}} APIs, and the authenticated principal must have the [IAM role(s)][predefined-iam-roles] required to access GCP resources using the {{metadata['repo']['name_pretty']}} API calls. + +## Getting Started + +### Prerequisites + +You will need a [Google Cloud Platform Console][developer-console] project with the {{metadata['repo']['name_pretty']}} [API enabled][enable-api]. +{% if metadata['repo']['requires_billing'] %}You will need to [enable billing][enable-billing] to use Google {{metadata['repo']['name_pretty']}}.{% endif %} +[Follow these instructions][create-project] to get your project set up. You will also need to set up the local development environment by +[installing the Google Cloud Command Line Interface][cloud-cli] and running the following commands in command line: +`gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +### Installation and setup + +You'll need to obtain the `{{ artifact_id }}` library. See the [Quickstart](#quickstart) section +to add `{{ artifact_id }}` as a dependency in your code. + +## About {{metadata['repo']['name_pretty']}} + +{% if 'partials' in metadata and metadata['partials']['about'] -%} +{{ metadata['partials']['about'] }} +{% else %} +[{{ metadata['repo']['name_pretty'] }}][product-docs] {{ metadata['repo']['api_description'] }} + +See the [{{metadata['repo']['name_pretty']}} client library docs][javadocs] to learn how to +use this {{metadata['repo']['name_pretty']}} Client Library. +{% endif %} + +{% if 'partials' in metadata and metadata['partials']['custom_content'] -%} +{{ metadata['partials']['custom_content'] }} +{% endif %} + +{% if metadata['samples']|length %} +## Samples + +Samples are in the [`samples/`](https://github.com/{{ metadata['repo']['repo'] }}/tree/main/samples) directory. + +| Sample | Source Code | Try it | +| --------------------------- | --------------------------------- | ------ | +{% for sample in metadata['samples'] %}| {{ sample.title }} | [source code](https://github.com/{{ metadata['repo']['repo'] }}/blob/main/{{ sample.file }}) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/{{ metadata['repo']['repo'] }}&page=editor&open_in_editor={{ sample.file }}) | +{% endfor %} +{% endif %} + +## Troubleshooting + +To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. + +{% if metadata['repo']['transport'] -%} +## Transport + +{% if metadata['repo']['transport'] == 'grpc' -%} +{{metadata['repo']['name_pretty']}} uses gRPC for the transport layer. +{% elif metadata['repo']['transport'] == 'http' -%} +{{metadata['repo']['name_pretty']}} uses HTTP/JSON for the transport layer. +{% elif metadata['repo']['transport'] == 'both' -%} +{{metadata['repo']['name_pretty']}} uses both gRPC and HTTP/JSON for the transport layer. +{% endif %} +{% endif -%} + +## Supported Java Versions + +Java {{ metadata['min_java_version'] }} or above is required for using this client. + +Google's Java client libraries, +[Google Cloud Client Libraries][cloudlibs] +and +[Google Cloud API Libraries][apilibs], +follow the +[Oracle Java SE support roadmap][oracle] +(see the Oracle Java SE Product Releases section). + +### For new development + +In general, new feature development occurs with support for the lowest Java +LTS version covered by Oracle's Premier Support (which typically lasts 5 years +from initial General Availability). If the minimum required JVM for a given +library is changed, it is accompanied by a [semver][semver] major release. + +Java 11 and (in September 2021) Java 17 are the best choices for new +development. + +### Keeping production systems current + +Google tests its client libraries with all current LTS versions covered by +Oracle's Extended Support (which typically lasts 8 years from initial +General Availability). + +#### Legacy support + +Google's client libraries support legacy versions of Java runtimes with long +term stable libraries that don't receive feature updates on a best efforts basis +as it may not be possible to backport all patches. + +Google provides updates on a best efforts basis to apps that continue to use +Java 7, though apps might need to upgrade to current versions of the library +that supports their JVM. + +#### Where to find specific information + +The latest versions and the supported Java versions are identified on +the individual GitHub repository `github.com/GoogleAPIs/java-SERVICENAME` +and on [google-cloud-java][g-c-j]. + +## Versioning + +{% if 'partials' in metadata and metadata['partials']['versioning'] -%} +{{ metadata['partials']['versioning'] }} +{% else %} +This library follows [Semantic Versioning](http://semver.org/). + +{% if metadata['repo']['release_level'] in ['preview'] %} +It is currently in major version zero (``0.y.z``), which means that anything may change at any time +and the public API should not be considered stable. +{% endif %}{% endif %} + +## Contributing + +{% if 'partials' in metadata and metadata['partials']['contributing'] -%} +{{ metadata['partials']['contributing'] }} +{% else %} +Contributions to this library are always welcome and highly encouraged. + +See [CONTRIBUTING][contributing] for more information how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in +this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more +information. +{% endif %} + +## License + +Apache 2.0 - See [LICENSE][license] for more information. + +## CI Status + +Java Version | Status +------------ | ------{% if metadata['min_java_version'] <= 7 %} +Java 7 | [![Kokoro CI][kokoro-badge-image-1]][kokoro-badge-link-1]{% endif %} +Java 8 | [![Kokoro CI][kokoro-badge-image-2]][kokoro-badge-link-2] +Java 8 OSX | [![Kokoro CI][kokoro-badge-image-3]][kokoro-badge-link-3] +Java 8 Windows | [![Kokoro CI][kokoro-badge-image-4]][kokoro-badge-link-4] +Java 11 | [![Kokoro CI][kokoro-badge-image-5]][kokoro-badge-link-5] + +Java is a registered trademark of Oracle and/or its affiliates. + +[product-docs]: {{metadata['repo']['product_documentation']}} +[javadocs]: {{metadata['repo']['client_documentation']}} +[kokoro-badge-image-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java7.svg +[kokoro-badge-link-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java7.html +[kokoro-badge-image-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8.svg +[kokoro-badge-link-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8.html +[kokoro-badge-image-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-osx.svg +[kokoro-badge-link-3]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-osx.html +[kokoro-badge-image-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-win.svg +[kokoro-badge-link-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java8-win.html +[kokoro-badge-image-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.svg +[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.html +[stability-image]: https://img.shields.io/badge/stability-{% if metadata['repo']['release_level'] == 'stable' %}stable-green{% elif metadata['repo']['release_level'] == 'preview' %}preview-yellow{% else %}unknown-red{% endif %} +[maven-version-image]: https://img.shields.io/maven-central/v/{{ group_id }}/{{ artifact_id }}.svg +[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['latest_version'] }} +[authentication]: https://github.com/googleapis/google-cloud-java#authentication +[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes +[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles +[iam-policy]: https://cloud.google.com/iam/docs/overview#cloud-iam-policy +[developer-console]: https://console.developers.google.com/ +[create-project]: https://cloud.google.com/resource-manager/docs/creating-managing-projects +[cloud-cli]: https://cloud.google.com/cli +[troubleshooting]: https://github.com/googleapis/google-cloud-java/blob/main/TROUBLESHOOTING.md +[contributing]: https://github.com/{{metadata['repo']['repo']}}/blob/main/CONTRIBUTING.md +[code-of-conduct]: https://github.com/{{metadata['repo']['repo']}}/blob/main/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[license]: https://github.com/{{metadata['repo']['repo']}}/blob/main/LICENSE +{% if metadata['repo']['requires_billing'] %}[enable-billing]: https://cloud.google.com/apis/docs/getting-started#enabling_billing{% endif %} +{% if metadata['repo']['api_id'] %}[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid={{ metadata['repo']['api_id'] }}{% endif %} +[libraries-bom]: https://github.com/GoogleCloudPlatform/cloud-opensource-java/wiki/The-Google-Cloud-Platform-Libraries-BOM +[shell_img]: https://gstatic.com/cloudssh/images/open-btn.png + +[semver]: https://semver.org/ +[cloudlibs]: https://cloud.google.com/apis/docs/client-libraries-explained +[apilibs]: https://cloud.google.com/apis/docs/client-libraries-explained#google_api_client_libraries +[oracle]: https://www.oracle.com/java/technologies/java-se-support-roadmap.html +[g-c-j]: http://github.com/googleapis/google-cloud-java diff --git a/library_generation/owlbot/templates/java_library/SECURITY.md b/library_generation/owlbot/templates/java_library/SECURITY.md new file mode 100644 index 0000000000..8b58ae9c01 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). + +The Google Security Team will respond within 5 working days of your report on g.co/vulnz. + +We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. diff --git a/library_generation/owlbot/templates/java_library/java.header b/library_generation/owlbot/templates/java_library/java.header new file mode 100644 index 0000000000..d0970ba7d3 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/java.header @@ -0,0 +1,15 @@ +^/\*$ +^ \* Copyright \d\d\d\d,? Google (Inc\.|LLC)$ +^ \*$ +^ \* Licensed under the Apache License, Version 2\.0 \(the "License"\);$ +^ \* you may not use this file except in compliance with the License\.$ +^ \* You may obtain a copy of the License at$ +^ \*$ +^ \*[ ]+https?://www.apache.org/licenses/LICENSE-2\.0$ +^ \*$ +^ \* Unless required by applicable law or agreed to in writing, software$ +^ \* distributed under the License is distributed on an "AS IS" BASIS,$ +^ \* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied\.$ +^ \* See the License for the specific language governing permissions and$ +^ \* limitations under the License\.$ +^ \*/$ diff --git a/library_generation/owlbot/templates/java_library/license-checks.xml b/library_generation/owlbot/templates/java_library/license-checks.xml new file mode 100644 index 0000000000..6597fced80 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/license-checks.xml @@ -0,0 +1,10 @@ + + + + + + + + diff --git a/library_generation/owlbot/templates/java_library/renovate.json b/library_generation/owlbot/templates/java_library/renovate.json new file mode 100644 index 0000000000..16c68a2387 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/renovate.json @@ -0,0 +1,80 @@ +{% if migrated_split_repo %}{ + "enabled": false, +{% else %}{ +{% endif %} "extends": [ + ":separateMajorReleases", + ":combinePatchMinorReleases", + ":ignoreUnstable", + ":prImmediately", + ":updateNotScheduled", + ":automergeDisabled", + ":ignoreModulesAndTests", + ":maintainLockFilesDisabled", + ":autodetectPinVersions" + ], + "ignorePaths": [ + ".kokoro/requirements.txt", + ".github/workflows/**" + ], + "packageRules": [ + { + "packagePatterns": [ + "^com.google.guava:" + ], + "versionScheme": "docker" + }, + { + "packagePatterns": [ + "*" + ], + "semanticCommitType": "deps", + "semanticCommitScope": null + }, + { + "packagePatterns": [ + "^org.apache.maven", + "^org.jacoco:", + "^org.codehaus.mojo:", + "^org.sonatype.plugins:", + "^com.coveo:", + "^com.google.cloud:google-cloud-shared-config" + ], + "semanticCommitType": "build", + "semanticCommitScope": "deps" + }, + { + "packagePatterns": [ + "^{{metadata['repo']['distribution_name']}}", + "^com.google.cloud:libraries-bom", + "^com.google.cloud.samples:shared-configuration" + ], + "semanticCommitType": "chore", + "semanticCommitScope": "deps" + }, + { + "packagePatterns": [ + "^junit:junit", + "^com.google.truth:truth", + "^org.mockito:mockito-core", + "^org.objenesis:objenesis", + "^com.google.cloud:google-cloud-conformance-tests" + ], + "semanticCommitType": "test", + "semanticCommitScope": "deps" + }, + { + "packagePatterns": [ + "^com.google.cloud:google-cloud-" + ], + "ignoreUnstable": false + }, + { + "packagePatterns": [ + "^com.fasterxml.jackson.core" + ], + "groupName": "jackson dependencies" + } + ], + "semanticCommits": true, + "dependencyDashboard": true +} diff --git a/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml b/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml new file mode 100644 index 0000000000..110250d003 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml @@ -0,0 +1,86 @@ +{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} +{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} + + + 4.0.0 + com.google.cloud + {{metadata['repo']['name']}}-install-without-bom + jar + Google {{metadata['repo']['name_pretty']}} Install Without Bom + https://github.com/{{metadata['repo']['repo']}} + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + + + + {{ group_id }} + {{ artifact_id }} + {{ metadata['latest_version'] }} + + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + diff --git a/library_generation/owlbot/templates/java_library/samples/pom.xml b/library_generation/owlbot/templates/java_library/samples/pom.xml new file mode 100644 index 0000000000..0f11429996 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/samples/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + com.google.cloud + google-cloud-{{metadata['repo']['name']}}-samples + 0.0.1-SNAPSHOT + pom + Google {{metadata['repo']['name_pretty']}} Samples Parent + https://github.com/{{metadata['repo']['repo']}} + + Java idiomatic client for Google Cloud Platform services. + + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + install-without-bom + snapshot + snippets + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.13 + + true + + + + + diff --git a/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml b/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml new file mode 100644 index 0000000000..62a83b440e --- /dev/null +++ b/library_generation/owlbot/templates/java_library/samples/snapshot/pom.xml @@ -0,0 +1,85 @@ +{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} +{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} + + + 4.0.0 + com.google.cloud + {{metadata['repo']['name']}}-snapshot + jar + Google {{metadata['repo']['name_pretty']}} Snapshot Samples + https://github.com/{{metadata['repo']['repo']}} + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + + + {{ group_id }} + {{ artifact_id }} + {{ metadata['latest_version'] }} + + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.3.0 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + diff --git a/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml b/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml new file mode 100644 index 0000000000..c6b9981507 --- /dev/null +++ b/library_generation/owlbot/templates/java_library/samples/snippets/pom.xml @@ -0,0 +1,49 @@ +{% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} +{% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} + + + 4.0.0 + com.google.cloud + {{metadata['repo']['name']}}-snippets + jar + Google {{metadata['repo']['name_pretty']}} Snippets + https://github.com/{{metadata['repo']['repo']}} + + + + com.google.cloud.samples + shared-configuration + 1.2.0 + + + + 1.8 + 1.8 + UTF-8 + + + + + + {{ group_id }} + {{ artifact_id }} + {{ metadata['latest_version'] }} + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.1.3 + test + + + From 37af9106233218ac88f5c2725c75e838c51b6970 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 2 Feb 2024 22:36:40 +0000 Subject: [PATCH 02/30] use locally hosted templates --- library_generation/owlbot/bin/entrypoint.sh | 10 ++++-- .../owlbot/src/apply-repo-templates.py | 31 ++++++++++++++----- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 80c4037689..3f02d49090 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -29,15 +29,18 @@ versions_file=$2 # Runs template and etc in current working directory function processModule() { + monorepo=$1 + # apply repo templates + echo "Generating templates" if [ -f "owlbot.py" ] then # defaults to run owlbot.py - python3 "${scripts_root}/owlbot/src/apply-repo-templates.py" owlbot.py + python3 "${scripts_root}/owlbot/src/apply-repo-templates.py" owlbot.py "${monorepo}" fi # templates as well as retrieving files from owl-bot-staging - echo "Generating templates and retrieving files from owl-bot-staging directory..." + echo "Retrieving files from owl-bot-staging directory..." if [ -f "owlbot.py" ] then # defaults to run owlbot.py @@ -77,6 +80,7 @@ function processModule() { # monorepo folders have an .OwlBot.yaml file in the module folder (e.g. # java-asset/.OwlBot.yaml), whereas HW libraries have the yaml in # `.github/.OwlBot.yaml` +monorepo="false" if [[ -f "$(pwd)/.OwlBot.yaml" ]]; then monorepo="true" fi @@ -87,4 +91,4 @@ if [[ "${monorepo}" == "true" ]]; then mv temp owl-bot-staging fi -processModule +processModule monorepo diff --git a/library_generation/owlbot/src/apply-repo-templates.py b/library_generation/owlbot/src/apply-repo-templates.py index 215e0286b7..c823686b5d 100644 --- a/library_generation/owlbot/src/apply-repo-templates.py +++ b/library_generation/owlbot/src/apply-repo-templates.py @@ -8,29 +8,44 @@ from absl import app from synthtool.languages.java import common_templates from pathlib import Path +import re script_dir = os.path.dirname(os.path.realpath(__file__)) repo_templates_path = os.path.join(script_dir, '..', 'templates', 'java_library') -def apply_repo_templates(owlbot_py_path: str) -> None: +def apply_repo_templates(owlbot_py_path: str, monorepo: bool) -> None: excludes = [] with open(owlbot_py_path) as contents: - excludes += _parse_template_excludes(contents) + excludes += _parse_template_excludes(contents.read()) print(f'repo_templates_path: {repo_templates_path}') print(f'excludes: {excludes}') - common_templates(excludes=excludes, template_path=Path(repo_templates_path)) + common_templates( + excludes=excludes, + template_path=Path(repo_templates_path), + monorepo=monorepo + ) def _parse_template_excludes(owlbot_py_contents: str) -> str: - return [ 'README.md' ] - pass + print(f'owlbot_py_contents: {owlbot_py_contents}') + excludes = re.search( + 'java\.common_templates\(.*excludes=\[(.*)\]\)', + owlbot_py_contents, + re.MULTILINE | re.DOTALL + ) + if excludes is None: + raise ValueError('Could not parse owlbot.py exclusions') + raw_excludes = excludes.group(1).split(',') + result = map(lambda raw_exc: re.search('["\'](.*)["\']', raw_exc).group(1), raw_excludes) + return result def main(argv: Sequence[str]) -> None: - if len(argv) != 2: - raise app.UsageError("Usage: python apply-repo-templates.py [owlbot_py_path]") + if len(argv) != 3: + raise app.UsageError("Usage: python apply-repo-templates.py owlbot_py_path monorepo") owlbot_py_path = argv[1] - apply_repo_templates(owlbot_py_path) + monorepo = argv[2] + apply_repo_templates(owlbot_py_path, monorepo.lower() == 'true') From 9e9e743cf8880e41a35c0ee9711314a9fd88aeb8 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 2 Feb 2024 22:37:43 +0000 Subject: [PATCH 03/30] fix regex parsing --- ...o-templates.py => apply_repo_templates.py} | 18 +++++++---- library_generation/postprocess_library.sh | 3 ++ library_generation/test/unit_tests.py | 30 +++++++++++++++++-- 3 files changed, 42 insertions(+), 9 deletions(-) rename library_generation/owlbot/src/{apply-repo-templates.py => apply_repo_templates.py} (67%) diff --git a/library_generation/owlbot/src/apply-repo-templates.py b/library_generation/owlbot/src/apply_repo_templates.py similarity index 67% rename from library_generation/owlbot/src/apply-repo-templates.py rename to library_generation/owlbot/src/apply_repo_templates.py index c823686b5d..d37e71533a 100644 --- a/library_generation/owlbot/src/apply-repo-templates.py +++ b/library_generation/owlbot/src/apply_repo_templates.py @@ -1,6 +1,8 @@ -"""TODO(diegomarquezp): DO NOT SUBMIT without one-line documentation for apply-repo-templates. - -TODO(diegomarquezp): DO NOT SUBMIT without a detailed description of apply-repo-templates. +""" +This script parses an owlbot.py file, specifically the call to `java.common_templates` in +order to extract the excluded files so it can be called with a custom template path +pointing to the templates hosted in `sdk-platform-java/library_generation/owlbot/templates`. +Briefly, this wraps the call to synthtool's common templates using a custom template folder. """ import os @@ -26,17 +28,21 @@ def apply_repo_templates(owlbot_py_path: str, monorepo: bool) -> None: ) -def _parse_template_excludes(owlbot_py_contents: str) -> str: +def parse_template_excludes(owlbot_py_contents: str) -> str: print(f'owlbot_py_contents: {owlbot_py_contents}') excludes = re.search( - 'java\.common_templates\(.*excludes=\[(.*)\]\)', + 'java\.common_templates\(.*excludes=\[(.*)\].*\)', owlbot_py_contents, re.MULTILINE | re.DOTALL ) if excludes is None: raise ValueError('Could not parse owlbot.py exclusions') raw_excludes = excludes.group(1).split(',') - result = map(lambda raw_exc: re.search('["\'](.*)["\']', raw_exc).group(1), raw_excludes) + result = [] + for raw_exc in raw_excludes: + match = re.search('["\'](.*)["\']', raw_exc) + if match: + result.append(match.group(1)) return result def main(argv: Sequence[str]) -> None: diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index f7035ec6c8..5c325609e5 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -95,11 +95,14 @@ docker run --rm \ # we clone the synthtool library and manually build it mkdir -p /tmp/synthtool pushd /tmp/synthtool + if [ ! -d "synthtool" ]; then git clone https://github.com/googleapis/synthtool.git fi pushd "synthtool" + git reset --hard "${synthtool_commitish}" + python3 -m pip install -e . python3 -m pip install -r requirements.in popd # synthtool diff --git a/library_generation/test/unit_tests.py b/library_generation/test/unit_tests.py index 13d2eaacf9..4161be674a 100644 --- a/library_generation/test/unit_tests.py +++ b/library_generation/test/unit_tests.py @@ -1,5 +1,5 @@ """ -Unit tests for utilities.py +Unit tests for utilities.py and non-transferred owlbot scripts """ import unittest @@ -15,6 +15,7 @@ from model.GapicConfig import GapicConfig from model.GenerationConfig import GenerationConfig from model.ClientInputs import parse as parse_build_file +from owlbot.src.apply_repo_templates import parse_template_excludes script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, 'resources') @@ -181,8 +182,31 @@ def test_client_inputs_parse_no_service_yaml_returns_empty_string(self): parsed = parse_build_file(build_file, 'test/versioned/path', 'BUILD_no_service_yaml.bazel') self.assertEqual('', parsed.service_yaml) - - + def test_apply_repo_templates_parse_template_excludes_valid_input_succeeds(self): + expected_excludes = ['exclude1', 'exclude2/*', 'exclude-3'] + subcases = [""" + java.common_templates(excludes=['exclude1','exclude2/*','exclude-3']) + """,""" + java.common_templates( + excludes=[ + 'exclude1', + 'exclude2/*', + 'exclude-3' + ] + )""",""" + java.common_templates( + excludes=[ + "exclude1", + "exclude2/*", + "exclude-3", + + ] + )""" + ] + + for subcase in subcases: + result = parse_template_excludes(subcase) + self.assertEqual(expected_excludes, result) From d2a8ca403500f3af4646e36b3864abc52392b95d Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 2 Feb 2024 22:40:26 +0000 Subject: [PATCH 04/30] update protobuf and generator versions in IT --- .../integration/google-cloud-java/generation_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml index 7b73f329d0..4e046837eb 100644 --- a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml @@ -1,9 +1,9 @@ #Required. -gapic_generator_version: 2.32.0 +gapic_generator_version: 2.34.0 #Optional. # grpc_version: 1.60.0 #Optional. The protobuf version in googleapis (not sdk-platform-java) is the actual source of truth for generated protos in google-cloud-java -protobuf_version: 23.2 +protobuf_version: 25.2 #Required. googleapis_commitish: 4512234113a18c1fda1fb0d0ceac8f4b4efe9801 #Required. From 2cb0a3186055ebeb8347db1758724252e0baa556 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 6 Feb 2024 16:45:19 +0000 Subject: [PATCH 05/30] minor fixes --- library_generation/owlbot/bin/entrypoint.sh | 2 +- library_generation/owlbot/src/apply_repo_templates.py | 2 +- library_generation/test/generate_library_integration_test.sh | 2 +- .../integration/google-cloud-java/generation_config.yaml | 5 +++-- .../integration/java-bigtable/generation_config.yaml | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 3f02d49090..52cdb3fda0 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -36,7 +36,7 @@ function processModule() { if [ -f "owlbot.py" ] then # defaults to run owlbot.py - python3 "${scripts_root}/owlbot/src/apply-repo-templates.py" owlbot.py "${monorepo}" + python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" owlbot.py "${monorepo}" fi # templates as well as retrieving files from owl-bot-staging diff --git a/library_generation/owlbot/src/apply_repo_templates.py b/library_generation/owlbot/src/apply_repo_templates.py index d37e71533a..9b6ff6a909 100644 --- a/library_generation/owlbot/src/apply_repo_templates.py +++ b/library_generation/owlbot/src/apply_repo_templates.py @@ -18,7 +18,7 @@ def apply_repo_templates(owlbot_py_path: str, monorepo: bool) -> None: excludes = [] with open(owlbot_py_path) as contents: - excludes += _parse_template_excludes(contents.read()) + excludes += parse_template_excludes(contents.read()) print(f'repo_templates_path: {repo_templates_path}') print(f'excludes: {excludes}') common_templates( diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index 9b46304da3..2a70ac11f3 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -87,7 +87,7 @@ for configuration_yaml in "${configuration_yamls[@]}"; do generation_duration_seconds=$(expr "${generation_end}" - "${generation_start}" || true) echo "Generation time for ${api_shortname} was ${generation_duration_seconds} seconds." pushd "${output_folder}" - echo "${proto_path} ${generation_duration_seconds}" >> generation_times + echo "${api_shortname} ${generation_duration_seconds}" >> generation_times echo "Generate library finished." echo "Compare generation result..." diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml index 4e046837eb..3092887f32 100644 --- a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml @@ -5,11 +5,11 @@ gapic_generator_version: 2.34.0 #Optional. The protobuf version in googleapis (not sdk-platform-java) is the actual source of truth for generated protos in google-cloud-java protobuf_version: 25.2 #Required. -googleapis_commitish: 4512234113a18c1fda1fb0d0ceac8f4b4efe9801 +googleapis_commitish: 00b0ded37f3e6c64282730e782a8c95c4f9be4bd #Required. owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 #Required. -synthtool_commitish: fac8444edd5f5526e804c306b766a271772a3e2f +synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e #Required. The root folder name of generated client libraries. destination_path: google-cloud-java #Required. If the number of libraries is greater than 1, the scripts will treat the target repository as a monorepo, with a slightly different workflow mainly in the postprocessing stage @@ -66,6 +66,7 @@ libraries: GAPICs: - proto_path: google/cloud/compute/v1 - api_shortname: kms + googleapis_commitish: 667eaac011c8a0096241ae4d27061e212af16540 library_type: GAPIC_AUTO GAPICs: - proto_path: google/cloud/kms/v1 diff --git a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml index 4a82a3e2c4..1952ca29c4 100644 --- a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml +++ b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml @@ -1,6 +1,6 @@ gapic_generator_version: 2.32.0 grpc_version: 1.61.0 -protobuf_version: 23.2 +protobuf_version: 25.2 googleapis_commitish: 4512234113a18c1fda1fb0d0ceac8f4b4efe9801 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 From e1c262dc6d38a3203e51abcdb1d9731408a3525e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 6 Feb 2024 16:57:46 +0000 Subject: [PATCH 06/30] install synthtool for unit tests --- .github/workflows/verify_library_generation.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index 0b4ae1b8ed..5073cf21d2 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -98,6 +98,21 @@ jobs: pushd library_generation pip install -r requirements.in popd + - name: install synthtool + shell: bash + run: | + set -ex + mkdir -p /tmp/synthtool + pushd /tmp/synthtool + if [ ! -d "synthtool" ]; then + git clone https://github.com/googleapis/synthtool.git + fi + pushd "synthtool" + + git reset --hard origin/no-java-templates + + python3 -m pip install -e . + python3 -m pip install -r requirements.in - name: Run shell unit tests run: | set -x From 79b41bc34da8564258025067bfacac710740457f Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 6 Feb 2024 18:18:14 +0000 Subject: [PATCH 07/30] use repo level googleapis commitish for kms --- .../integration/google-cloud-java/generation_config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml index 3092887f32..3e56eaf0b4 100644 --- a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml @@ -66,7 +66,6 @@ libraries: GAPICs: - proto_path: google/cloud/compute/v1 - api_shortname: kms - googleapis_commitish: 667eaac011c8a0096241ae4d27061e212af16540 library_type: GAPIC_AUTO GAPICs: - proto_path: google/cloud/kms/v1 From 7d6dabc32765eccf53f6280e1bd8cf68b4e8f17e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 6 Feb 2024 18:20:16 +0000 Subject: [PATCH 08/30] comment fixes --- library_generation/owlbot/bin/entrypoint.sh | 1 - library_generation/test/unit_tests.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 52cdb3fda0..89cc1f59b9 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -35,7 +35,6 @@ function processModule() { echo "Generating templates" if [ -f "owlbot.py" ] then - # defaults to run owlbot.py python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" owlbot.py "${monorepo}" fi diff --git a/library_generation/test/unit_tests.py b/library_generation/test/unit_tests.py index 4161be674a..24996572b7 100644 --- a/library_generation/test/unit_tests.py +++ b/library_generation/test/unit_tests.py @@ -1,5 +1,5 @@ """ -Unit tests for utilities.py and non-transferred owlbot scripts +Unit tests for python scripts """ import unittest From 7c503c9e5b36e94285c24eaf394246fdca52543f Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Tue, 6 Feb 2024 14:18:44 -0500 Subject: [PATCH 09/30] Update library_generation/owlbot/bin/entrypoint.sh Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> --- library_generation/owlbot/bin/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 89cc1f59b9..f0bcc684cd 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -32,7 +32,7 @@ function processModule() { monorepo=$1 # apply repo templates - echo "Generating templates" + echo "Rendering templates" if [ -f "owlbot.py" ] then python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" owlbot.py "${monorepo}" From 5587d0f5b1d2baf3e3ce7d25117219eb323161f7 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 7 Feb 2024 18:34:52 +0000 Subject: [PATCH 10/30] use config yaml to compute exclusions --- library_generation/__init__.py | 0 .../generate_composed_library.py | 2 +- library_generation/model/GenerationConfig.py | 6 +++ library_generation/model/__init__.py | 0 library_generation/owlbot/bin/entrypoint.sh | 10 ++++- library_generation/owlbot/src/__init__.py | 0 .../owlbot/src/apply_repo_templates.py | 42 ++++++------------- library_generation/owlbot/src/requirements.in | 2 - library_generation/postprocess_library.sh | 5 ++- library_generation/setup.py | 14 +++++++ .../google-cloud-java/generation_config.yaml | 13 ++++++ .../java-bigtable/generation_config.yaml | 18 +++++++- 12 files changed, 77 insertions(+), 35 deletions(-) create mode 100644 library_generation/__init__.py create mode 100644 library_generation/model/__init__.py create mode 100644 library_generation/owlbot/src/__init__.py create mode 100644 library_generation/setup.py diff --git a/library_generation/__init__.py b/library_generation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index d5beec733b..c8dbb7b8fc 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -137,5 +137,5 @@ def generate_composed_library( # call postprocess library util.run_process_and_print_output([f'{script_dir}/postprocess_library.sh', f'{library_path}', '', versions_file, owlbot_cli_source_folder, - config.owlbot_cli_image, config.synthtool_commitish, str(is_monorepo).lower()], 'Library postprocessing') + config.owlbot_cli_image, config.synthtool_commitish, str(is_monorepo).lower(), config.path_to_yaml], 'Library postprocessing') diff --git a/library_generation/model/GenerationConfig.py b/library_generation/model/GenerationConfig.py index 77273b10eb..a2ad972839 100644 --- a/library_generation/model/GenerationConfig.py +++ b/library_generation/model/GenerationConfig.py @@ -17,6 +17,8 @@ def __init__( owlbot_cli_image: str, synthtool_commitish: str, destination_path: Optional[str], + template_excludes: str, + path_to_yaml: str, libraries: List[LibraryConfig], ): self.gapic_generator_version = gapic_generator_version @@ -26,6 +28,8 @@ def __init__( self.owlbot_cli_image = owlbot_cli_image self.synthtool_commitish = synthtool_commitish self.destination_path = destination_path + self.template_excludes = template_excludes + self.path_to_yaml = path_to_yaml self.libraries = libraries """ @@ -75,6 +79,8 @@ def from_yaml(path_to_yaml: str): _required(config, 'owlbot_cli_image'), _required(config, 'synthtool_commitish'), _optional(config, 'destination_path', None), + _required(config, 'template_excludes'), + path_to_yaml, parsed_libraries ) diff --git a/library_generation/model/__init__.py b/library_generation/model/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 89cc1f59b9..1addcfbfd4 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -26,6 +26,14 @@ set -ex scripts_root=$1 versions_file=$2 +configuration_yaml=$3 + +pushd "${scripts_root}" +# install library_generation which contains model classes used by the the +# template rendering step +python -m pip install -e . +popd #scripts_root + # Runs template and etc in current working directory function processModule() { @@ -35,7 +43,7 @@ function processModule() { echo "Generating templates" if [ -f "owlbot.py" ] then - python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" owlbot.py "${monorepo}" + python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}" fi # templates as well as retrieving files from owl-bot-staging diff --git a/library_generation/owlbot/src/__init__.py b/library_generation/owlbot/src/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/library_generation/owlbot/src/apply_repo_templates.py b/library_generation/owlbot/src/apply_repo_templates.py index 9b6ff6a909..fc450927b5 100644 --- a/library_generation/owlbot/src/apply_repo_templates.py +++ b/library_generation/owlbot/src/apply_repo_templates.py @@ -6,54 +6,38 @@ """ import os +import sys from collections.abc import Sequence -from absl import app from synthtool.languages.java import common_templates from pathlib import Path -import re +from model.GenerationConfig import GenerationConfig script_dir = os.path.dirname(os.path.realpath(__file__)) repo_templates_path = os.path.join(script_dir, '..', 'templates', 'java_library') -def apply_repo_templates(owlbot_py_path: str, monorepo: bool) -> None: - excludes = [] - with open(owlbot_py_path) as contents: - excludes += parse_template_excludes(contents.read()) +def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None: + config = GenerationConfig.from_yaml(configuration_yaml_path) print(f'repo_templates_path: {repo_templates_path}') - print(f'excludes: {excludes}') + print(f'excludes: {config.template_excludes}') common_templates( - excludes=excludes, + excludes=config.template_excludes, template_path=Path(repo_templates_path), monorepo=monorepo ) -def parse_template_excludes(owlbot_py_contents: str) -> str: - print(f'owlbot_py_contents: {owlbot_py_contents}') - excludes = re.search( - 'java\.common_templates\(.*excludes=\[(.*)\].*\)', - owlbot_py_contents, - re.MULTILINE | re.DOTALL - ) - if excludes is None: - raise ValueError('Could not parse owlbot.py exclusions') - raw_excludes = excludes.group(1).split(',') - result = [] - for raw_exc in raw_excludes: - match = re.search('["\'](.*)["\']', raw_exc) - if match: - result.append(match.group(1)) - return result - def main(argv: Sequence[str]) -> None: if len(argv) != 3: - raise app.UsageError("Usage: python apply-repo-templates.py owlbot_py_path monorepo") + raise ValueError("Usage: python apply-repo-templates.py configuration_yaml_path monorepo") - owlbot_py_path = argv[1] + configuration_yaml_path = argv[1] monorepo = argv[2] - apply_repo_templates(owlbot_py_path, monorepo.lower() == 'true') + apply_repo_templates( + configuration_yaml_path, + monorepo.lower() == 'true' + ) if __name__ == "__main__": - app.run(main) + main(sys.argv) diff --git a/library_generation/owlbot/src/requirements.in b/library_generation/owlbot/src/requirements.in index a4b12718fa..fed3e32d94 100644 --- a/library_generation/owlbot/src/requirements.in +++ b/library_generation/owlbot/src/requirements.in @@ -8,5 +8,3 @@ colorlog protobuf watchdog requests -pyyaml -absl-py diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 5c325609e5..9f7c2efa0a 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -20,6 +20,8 @@ # provided # 7 - is_monorepo: whether this library is a monorepo, which implies slightly # different logic +# 8 - configuration_yaml_path: path to the configuration yaml containing library +# generation information for this library set -xeo pipefail scripts_root=$(dirname "$(readlink -f "$0")") @@ -30,6 +32,7 @@ owlbot_cli_source_folder=$4 owlbot_cli_image_sha=$5 synthtool_commitish=$6 is_monorepo=$7 +configuration_yaml_path=$8 source "${scripts_root}"/utilities.sh @@ -116,5 +119,5 @@ popd # owlbot/src # run the postprocessor echo 'running owl-bot post-processor' pushd "${postprocessing_target}" -bash "${scripts_root}/owlbot/bin/entrypoint.sh" "${scripts_root}" "${versions_file}" +bash "${scripts_root}/owlbot/bin/entrypoint.sh" "${scripts_root}" "${versions_file}" "${configuration_yaml_path}" popd # postprocessing_target diff --git a/library_generation/setup.py b/library_generation/setup.py new file mode 100644 index 0000000000..de0ed02d08 --- /dev/null +++ b/library_generation/setup.py @@ -0,0 +1,14 @@ +"""TODO(diegomarquezp): DO NOT SUBMIT without one-line documentation for setup. + +TODO(diegomarquezp): DO NOT SUBMIT without a detailed description of setup. +""" + +from distutils.core import setup + +setup(name='sdk_platform_java_library_generation', + version='0.1', + py_modules=[ + 'model', + 'owlbot.src' + ], + ) diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml index 3e56eaf0b4..f109df15dc 100644 --- a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml @@ -12,6 +12,19 @@ owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1c synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e #Required. The root folder name of generated client libraries. destination_path: google-cloud-java +#Required. This indicates which template files should not be added to the target library in the postprocessing step. This is originally expressed in owlbot.py. Since google-cloud-java has the same exclusion list for all libraries, this will be a repo-level configuration (for now, while we don't find exceptions). +template_excludes: + - ".github/*" + - ".kokoro/*" + - "samples/*" + - "CODE_OF_CONDUCT.md" + - "CONTRIBUTING.md" + - "LICENSE" + - "SECURITY.md" + - "java.header" + - "license-checks.xml" + - "renovate.json" + - ".gitignore" #Required. If the number of libraries is greater than 1, the scripts will treat the target repository as a monorepo, with a slightly different workflow mainly in the postprocessing stage libraries: #Required. Can be used for populating the folder name java-{api_shortName}. This is also the destination-name in new-client.py. diff --git a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml index 1952ca29c4..057f90a936 100644 --- a/library_generation/test/resources/integration/java-bigtable/generation_config.yaml +++ b/library_generation/test/resources/integration/java-bigtable/generation_config.yaml @@ -1,10 +1,26 @@ gapic_generator_version: 2.32.0 grpc_version: 1.61.0 protobuf_version: 25.2 -googleapis_commitish: 4512234113a18c1fda1fb0d0ceac8f4b4efe9801 +googleapis_commitish: 40203ca1880849480bbff7b8715491060bbccdf1 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5 destination_path: java-bigtable +template_excludes: + - ".gitignore" + - ".kokoro/presubmit/integration.cfg" + - ".kokoro/presubmit/graalvm-native.cfg" + - ".kokoro/presubmit/graalvm-native-17.cfg" + - ".kokoro/nightly/integration.cfg" + - ".kokoro/presubmit/samples.cfg" + - ".kokoro/nightly/samples.cfg" + - ".github/ISSUE_TEMPLATE/bug_report.md" + - ".github/PULL_REQUEST_TEMPLATE.md" + - "CONTRIBUTING.md" + - "codecov.yaml" + - ".github/release-please.yml" + - "renovate.json" + - ".kokoro/requirements.in" + - ".kokoro/requirements.txt" libraries: - api_shortname: bigtable name_pretty: Cloud Bigtable From 65d8ef923a7c0a5055569742d768f2d6289d3c1b Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 7 Feb 2024 18:37:35 +0000 Subject: [PATCH 11/30] remove unused __init__.py files --- library_generation/__init__.py | 0 library_generation/model/__init__.py | 0 library_generation/owlbot/src/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 library_generation/__init__.py delete mode 100644 library_generation/model/__init__.py delete mode 100644 library_generation/owlbot/src/__init__.py diff --git a/library_generation/__init__.py b/library_generation/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/model/__init__.py b/library_generation/model/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/library_generation/owlbot/src/__init__.py b/library_generation/owlbot/src/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From b980b88743fbab1aaf78513a4d2764f716f2cdd5 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 7 Feb 2024 18:41:12 +0000 Subject: [PATCH 12/30] remove owlbot.py check in template rendering postprocessing step --- library_generation/owlbot/bin/entrypoint.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 516d803a70..f76f6fc67b 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -41,10 +41,7 @@ function processModule() { # apply repo templates echo "Rendering templates" - if [ -f "owlbot.py" ] - then - python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}" - fi + python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}" # templates as well as retrieving files from owl-bot-staging echo "Retrieving files from owl-bot-staging directory..." From f2cd5bf58522c6a09ad6609664ca5102e1dccc6c Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 7 Feb 2024 18:43:07 +0000 Subject: [PATCH 13/30] add comment for setup.py --- library_generation/setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/library_generation/setup.py b/library_generation/setup.py index de0ed02d08..7e9a9a0470 100644 --- a/library_generation/setup.py +++ b/library_generation/setup.py @@ -1,6 +1,5 @@ -"""TODO(diegomarquezp): DO NOT SUBMIT without one-line documentation for setup. - -TODO(diegomarquezp): DO NOT SUBMIT without a detailed description of setup. +""" +Package information of library_generation python scripts """ from distutils.core import setup From f416b4c622d78fe6bff9e31389af85c3ffffd52e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 7 Feb 2024 18:44:20 +0000 Subject: [PATCH 14/30] remove unit test --- library_generation/test/unit_tests.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/library_generation/test/unit_tests.py b/library_generation/test/unit_tests.py index 24996572b7..74f7ee569c 100644 --- a/library_generation/test/unit_tests.py +++ b/library_generation/test/unit_tests.py @@ -15,7 +15,6 @@ from model.GapicConfig import GapicConfig from model.GenerationConfig import GenerationConfig from model.ClientInputs import parse as parse_build_file -from owlbot.src.apply_repo_templates import parse_template_excludes script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, 'resources') @@ -182,32 +181,6 @@ def test_client_inputs_parse_no_service_yaml_returns_empty_string(self): parsed = parse_build_file(build_file, 'test/versioned/path', 'BUILD_no_service_yaml.bazel') self.assertEqual('', parsed.service_yaml) - def test_apply_repo_templates_parse_template_excludes_valid_input_succeeds(self): - expected_excludes = ['exclude1', 'exclude2/*', 'exclude-3'] - subcases = [""" - java.common_templates(excludes=['exclude1','exclude2/*','exclude-3']) - """,""" - java.common_templates( - excludes=[ - 'exclude1', - 'exclude2/*', - 'exclude-3' - ] - )""",""" - java.common_templates( - excludes=[ - "exclude1", - "exclude2/*", - "exclude-3", - - ] - )""" - ] - - for subcase in subcases: - result = parse_template_excludes(subcase) - self.assertEqual(expected_excludes, result) - if __name__ == "__main__": From 8d348a4289aef7735ae846401dc39c247e6b71d6 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 7 Feb 2024 18:52:58 +0000 Subject: [PATCH 15/30] simplify module processing script --- library_generation/owlbot/bin/entrypoint.sh | 91 ++++++++++----------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index f76f6fc67b..28cc9dce6a 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -34,51 +34,6 @@ pushd "${scripts_root}" python -m pip install -e . popd #scripts_root - -# Runs template and etc in current working directory -function processModule() { - monorepo=$1 - - # apply repo templates - echo "Rendering templates" - python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}" - - # templates as well as retrieving files from owl-bot-staging - echo "Retrieving files from owl-bot-staging directory..." - if [ -f "owlbot.py" ] - then - # defaults to run owlbot.py - python3 owlbot.py - fi - echo "...done" - - # write or restore pom.xml files - echo "Generating missing pom.xml..." - python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true" - echo "...done" - - # write or restore clirr-ignored-differences.xml - echo "Generating clirr-ignored-differences.xml..." - ${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}" - echo "...done" - - # fix license headers - echo "Fixing missing license headers..." - python3 "${scripts_root}/owlbot/src/fix-license-headers.py" - echo "...done" - - # TODO: re-enable this once we resolve thrashing - # restore license headers years - # echo "Restoring copyright years..." - # /owlbot/bin/restore_license_headers.sh - # echo "...done" - - # ensure formatting on all .java files in the repository - echo "Reformatting source..." - mvn fmt:format - echo "...done" -} - # This script can be used to process HW libraries and monorepo # (google-cloud-java) libraries, which require a slightly different treatment # monorepo folders have an .OwlBot.yaml file in the module folder (e.g. @@ -95,4 +50,48 @@ if [[ "${monorepo}" == "true" ]]; then mv temp owl-bot-staging fi -processModule monorepo + +# Runs template and etc in current working directory +monorepo=$1 + +# apply repo templates +echo "Rendering templates" +python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}" + +# templates as well as retrieving files from owl-bot-staging +echo "Retrieving files from owl-bot-staging directory..." +if [ -f "owlbot.py" ] +then + # defaults to run owlbot.py + python3 owlbot.py +fi +echo "...done" + +# write or restore pom.xml files +echo "Generating missing pom.xml..." +python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true" +echo "...done" + +# write or restore clirr-ignored-differences.xml +echo "Generating clirr-ignored-differences.xml..." +${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}" +echo "...done" + +# fix license headers +echo "Fixing missing license headers..." +python3 "${scripts_root}/owlbot/src/fix-license-headers.py" +echo "...done" + +# TODO: re-enable this once we resolve thrashing +# restore license headers years +# echo "Restoring copyright years..." +# /owlbot/bin/restore_license_headers.sh +# echo "...done" + +# ensure formatting on all .java files in the repository +echo "Reformatting source..." +mvn fmt:format +echo "...done" + + + From 9c778f6f10ce8a34394c096735748cdfc0b7db3b Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 7 Feb 2024 19:08:26 +0000 Subject: [PATCH 16/30] update googleapis commitish --- .../integration/google-cloud-java/generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml index f109df15dc..4a0a8fd456 100644 --- a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml @@ -5,7 +5,7 @@ gapic_generator_version: 2.34.0 #Optional. The protobuf version in googleapis (not sdk-platform-java) is the actual source of truth for generated protos in google-cloud-java protobuf_version: 25.2 #Required. -googleapis_commitish: 00b0ded37f3e6c64282730e782a8c95c4f9be4bd +googleapis_commitish: bfa99682e3fd1cf14e2b8a4945942ec7cf984f96 #Required. owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 #Required. From dbecc0e69f40e457c7190e4fa27eeba05253b7ed Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 7 Feb 2024 19:35:20 +0000 Subject: [PATCH 17/30] update googleapis commitish ii --- .../integration/google-cloud-java/generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml index 4a0a8fd456..828f52dbec 100644 --- a/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml +++ b/library_generation/test/resources/integration/google-cloud-java/generation_config.yaml @@ -5,7 +5,7 @@ gapic_generator_version: 2.34.0 #Optional. The protobuf version in googleapis (not sdk-platform-java) is the actual source of truth for generated protos in google-cloud-java protobuf_version: 25.2 #Required. -googleapis_commitish: bfa99682e3fd1cf14e2b8a4945942ec7cf984f96 +googleapis_commitish: eeb00f98eaa96ce7a58f2558b702141354f74ff0 #Required. owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409 #Required. From 2539e1e5a8146ccfaa53ddb4c7bede111dc6c863 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 18:54:25 +0000 Subject: [PATCH 18/30] post-merge fixes --- library_generation/model/generation_config.py | 6 ++++++ library_generation/owlbot/bin/entrypoint.sh | 6 ------ library_generation/owlbot/src/apply_repo_templates.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index 1c3c62d1fb..93bb057d7e 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -31,6 +31,8 @@ def __init__( googleapis_commitish: str, owlbot_cli_image: str, synthtool_commitish: str, + template_excludes: str, + path_to_yaml: str, libraries: List[LibraryConfig], grpc_version: Optional[str] = None, protobuf_version: Optional[str] = None, @@ -39,6 +41,8 @@ def __init__( self.googleapis_commitish = googleapis_commitish self.owlbot_cli_image = owlbot_cli_image self.synthtool_commitish = synthtool_commitish + self.template_excludes = template_excludes + self.path_to_yaml = path_to_yaml self.libraries = libraries self.grpc_version = grpc_version self.protobuf_version = protobuf_version @@ -94,6 +98,8 @@ def from_yaml(path_to_yaml: str): googleapis_commitish=__required(config, "googleapis_commitish"), owlbot_cli_image=__required(config, "owlbot_cli_image"), synthtool_commitish=__required(config, "synthtool_commitish"), + template_excludes=__required(config, 'template_excludes'), + path_to_yaml=path_to_yaml, libraries=parsed_libraries, ) diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 28cc9dce6a..a8d5a730e3 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -28,12 +28,6 @@ scripts_root=$1 versions_file=$2 configuration_yaml=$3 -pushd "${scripts_root}" -# install library_generation which contains model classes used by the the -# template rendering step -python -m pip install -e . -popd #scripts_root - # This script can be used to process HW libraries and monorepo # (google-cloud-java) libraries, which require a slightly different treatment # monorepo folders have an .OwlBot.yaml file in the module folder (e.g. diff --git a/library_generation/owlbot/src/apply_repo_templates.py b/library_generation/owlbot/src/apply_repo_templates.py index fc450927b5..7b6a463cc6 100644 --- a/library_generation/owlbot/src/apply_repo_templates.py +++ b/library_generation/owlbot/src/apply_repo_templates.py @@ -10,13 +10,13 @@ from collections.abc import Sequence from synthtool.languages.java import common_templates from pathlib import Path -from model.GenerationConfig import GenerationConfig +from library_generation.model.generation_config import from_yaml script_dir = os.path.dirname(os.path.realpath(__file__)) repo_templates_path = os.path.join(script_dir, '..', 'templates', 'java_library') def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None: - config = GenerationConfig.from_yaml(configuration_yaml_path) + config = from_yaml(configuration_yaml_path) print(f'repo_templates_path: {repo_templates_path}') print(f'excludes: {config.template_excludes}') common_templates( From 560644cafe3a61fd7932e81c016e483f8fc6f0d8 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 18:55:44 +0000 Subject: [PATCH 19/30] fix unit tests --- library_generation/test/unit_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library_generation/test/unit_tests.py b/library_generation/test/unit_tests.py index ebac87991e..540f180eff 100644 --- a/library_generation/test/unit_tests.py +++ b/library_generation/test/unit_tests.py @@ -357,6 +357,8 @@ def __get_a_gen_config(num: int): googleapis_commitish="", owlbot_cli_image="", synthtool_commitish="", + template_excludes=[], + path_to_yaml='.', libraries=libraries, ) From e0aade6ba684029fed4b9ee5bf83daeec0f1642f Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 18:57:34 +0000 Subject: [PATCH 20/30] delete generated goldens --- .../resources/goldens/.OwlBot-golden.yaml | 35 ------------------ .../goldens/.repo-metadata-golden.json | 18 ---------- .../resources/goldens/.repo-metadata.json | 18 ---------- .../test/resources/goldens/owlbot-golden.py | 36 ------------------- 4 files changed, 107 deletions(-) delete mode 100644 library_generation/test/resources/goldens/.OwlBot-golden.yaml delete mode 100644 library_generation/test/resources/goldens/.repo-metadata-golden.json delete mode 100644 library_generation/test/resources/goldens/.repo-metadata.json delete mode 100644 library_generation/test/resources/goldens/owlbot-golden.py diff --git a/library_generation/test/resources/goldens/.OwlBot-golden.yaml b/library_generation/test/resources/goldens/.OwlBot-golden.yaml deleted file mode 100644 index 225b4620bf..0000000000 --- a/library_generation/test/resources/goldens/.OwlBot-golden.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -deep-remove-regex: -- "/java-bare-metal-solution/grpc-google-.*/src" -- "/java-bare-metal-solution/proto-google-.*/src" -- "/java-bare-metal-solution/google-.*/src" -- "/java-bare-metal-solution/samples/snippets/generated" - -deep-preserve-regex: -- "/java-bare-metal-solution/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" - -deep-copy-regex: -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/proto-google-cloud-bare-metal-solution-$1/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/grpc-google-cloud-bare-metal-solution-$1/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/google-cloud-bare-metal-solution/src" -- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/samples/snippets/generated" - dest: "/owl-bot-staging/java-bare-metal-solution/$1/samples/snippets/generated" - -api-name: baremetalsolution \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata-golden.json b/library_generation/test/resources/goldens/.repo-metadata-golden.json deleted file mode 100644 index 88ee68b2e1..0000000000 --- a/library_generation/test/resources/goldens/.repo-metadata-golden.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "api_id": "baremetalsolution.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc" -} \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata.json b/library_generation/test/resources/goldens/.repo-metadata.json deleted file mode 100644 index 88ee68b2e1..0000000000 --- a/library_generation/test/resources/goldens/.repo-metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "api_shortname": "baremetalsolution", - "name_pretty": "Bare Metal Solution", - "product_documentation": "https://cloud.google.com/bare-metal/docs", - "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", - "release_level": "preview", - "transport": "grpc", - "language": "java", - "repo": "googleapis/google-cloud-java", - "repo_short": "java-bare-metal-solution", - "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", - "api_id": "baremetalsolution.googleapis.com", - "library_type": "GAPIC_AUTO", - "requires_billing": true, - "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc" -} \ No newline at end of file diff --git a/library_generation/test/resources/goldens/owlbot-golden.py b/library_generation/test/resources/goldens/owlbot-golden.py deleted file mode 100644 index c2c142892a..0000000000 --- a/library_generation/test/resources/goldens/owlbot-golden.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import synthtool as s -from synthtool.languages import java - - -for library in s.get_staging_dirs(): - # put any special-case replacements here - s.move(library) - -s.remove_staging_dirs() -java.common_templates(monorepo=True, excludes=[ - ".github/*", - ".kokoro/*", - "samples/*", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.md", - "LICENSE", - "SECURITY.md", - "java.header", - "license-checks.xml", - "renovate.json", - ".gitignore", -]) \ No newline at end of file From 1663a0f4deeee12b5a465ffc71f50683b0bcbf12 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 19:00:05 +0000 Subject: [PATCH 21/30] restore necessary golden files --- .../resources/goldens/.OwlBot-golden.yaml | 35 ++++++++++++++++++ .../goldens/.repo-metadata-golden.json | 18 ++++++++++ .../test/resources/goldens/owlbot-golden.py | 36 +++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 library_generation/test/resources/goldens/.OwlBot-golden.yaml create mode 100644 library_generation/test/resources/goldens/.repo-metadata-golden.json create mode 100644 library_generation/test/resources/goldens/owlbot-golden.py diff --git a/library_generation/test/resources/goldens/.OwlBot-golden.yaml b/library_generation/test/resources/goldens/.OwlBot-golden.yaml new file mode 100644 index 0000000000..225b4620bf --- /dev/null +++ b/library_generation/test/resources/goldens/.OwlBot-golden.yaml @@ -0,0 +1,35 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +deep-remove-regex: +- "/java-bare-metal-solution/grpc-google-.*/src" +- "/java-bare-metal-solution/proto-google-.*/src" +- "/java-bare-metal-solution/google-.*/src" +- "/java-bare-metal-solution/samples/snippets/generated" + +deep-preserve-regex: +- "/java-bare-metal-solution/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" + +deep-copy-regex: +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/proto-google-cloud-bare-metal-solution-$1/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/grpc-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/grpc-google-cloud-bare-metal-solution-$1/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/gapic-google-.*/src" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/google-cloud-bare-metal-solution/src" +- source: "/google/cloud/baremetalsolution/(v.*)/.*-java/samples/snippets/generated" + dest: "/owl-bot-staging/java-bare-metal-solution/$1/samples/snippets/generated" + +api-name: baremetalsolution \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata-golden.json b/library_generation/test/resources/goldens/.repo-metadata-golden.json new file mode 100644 index 0000000000..88ee68b2e1 --- /dev/null +++ b/library_generation/test/resources/goldens/.repo-metadata-golden.json @@ -0,0 +1,18 @@ +{ + "api_shortname": "baremetalsolution", + "name_pretty": "Bare Metal Solution", + "product_documentation": "https://cloud.google.com/bare-metal/docs", + "api_description": "Bring your Oracle workloads to Google Cloud with Bare Metal Solution and jumpstart your cloud journey with minimal risk.", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-bare-metal-solution/latest/overview", + "release_level": "preview", + "transport": "grpc", + "language": "java", + "repo": "googleapis/google-cloud-java", + "repo_short": "java-bare-metal-solution", + "distribution_name": "com.google.cloud:google-cloud-bare-metal-solution", + "api_id": "baremetalsolution.googleapis.com", + "library_type": "GAPIC_AUTO", + "requires_billing": true, + "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc" +} \ No newline at end of file diff --git a/library_generation/test/resources/goldens/owlbot-golden.py b/library_generation/test/resources/goldens/owlbot-golden.py new file mode 100644 index 0000000000..c2c142892a --- /dev/null +++ b/library_generation/test/resources/goldens/owlbot-golden.py @@ -0,0 +1,36 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import synthtool as s +from synthtool.languages import java + + +for library in s.get_staging_dirs(): + # put any special-case replacements here + s.move(library) + +s.remove_staging_dirs() +java.common_templates(monorepo=True, excludes=[ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore", +]) \ No newline at end of file From 0fb4b7ed5e302db8b5fc798ec011a9733ac4f37a Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 19:02:22 +0000 Subject: [PATCH 22/30] snippet bot to ignore templates --- .github/snippet-bot.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml index c7ee6ca54b..e5c4e4fd82 100644 --- a/.github/snippet-bot.yml +++ b/.github/snippet-bot.yml @@ -4,3 +4,4 @@ ignoreFiles: - src/test/** - test/** - showcase/** + - library_generation/owlbot/templates/** From ccfee71a3a29261b6f1aaf79f976f62a558ca557 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 19:04:22 +0000 Subject: [PATCH 23/30] fix snippet bot ii --- .github/snippet-bot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml index e5c4e4fd82..760de3b73e 100644 --- a/.github/snippet-bot.yml +++ b/.github/snippet-bot.yml @@ -4,4 +4,4 @@ ignoreFiles: - src/test/** - test/** - showcase/** - - library_generation/owlbot/templates/** + - library_generation/owlbot/templates/**/*.xml From 07ea2a6c852d3a49e856fe3db3a147a6230ce21a Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 19:05:26 +0000 Subject: [PATCH 24/30] fix snippet bot iii --- .github/snippet-bot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml index 760de3b73e..2bd34f14af 100644 --- a/.github/snippet-bot.yml +++ b/.github/snippet-bot.yml @@ -4,4 +4,4 @@ ignoreFiles: - src/test/** - test/** - showcase/** - - library_generation/owlbot/templates/**/*.xml + - **/owlbot/templates/**/*.xml From b38f755f920ccfb6dcae78bfc0e59036a1d28306 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 19:07:20 +0000 Subject: [PATCH 25/30] use full path to conflicting pomxml --- .github/snippet-bot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml index 2bd34f14af..dc775f34d9 100644 --- a/.github/snippet-bot.yml +++ b/.github/snippet-bot.yml @@ -4,4 +4,4 @@ ignoreFiles: - src/test/** - test/** - showcase/** - - **/owlbot/templates/**/*.xml + - library_generation/owlbot/templates/java_library/samples/pom.xml From 0e8b19e7ec126808f755093c74cedeb3366b3f87 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 19:08:21 +0000 Subject: [PATCH 26/30] use full path to actual conflicting pomxml --- .github/snippet-bot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml index dc775f34d9..9458774ff0 100644 --- a/.github/snippet-bot.yml +++ b/.github/snippet-bot.yml @@ -4,4 +4,4 @@ ignoreFiles: - src/test/** - test/** - showcase/** - - library_generation/owlbot/templates/java_library/samples/pom.xml + - library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml From f4e78671318e91fa4a4285ea17484533d1cff675 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 19:24:29 +0000 Subject: [PATCH 27/30] lint --- library_generation/model/generation_config.py | 2 +- .../owlbot/src/apply_repo_templates.py | 37 +++++----- library_generation/owlbot/src/fix-poms.py | 72 +++++++++++-------- library_generation/owlbot/src/gen-template.py | 7 +- library_generation/setup.py | 33 ++++----- library_generation/test/integration_tests.py | 2 +- library_generation/test/unit_tests.py | 2 +- 7 files changed, 87 insertions(+), 68 deletions(-) diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index 93bb057d7e..7317edc932 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -98,7 +98,7 @@ def from_yaml(path_to_yaml: str): googleapis_commitish=__required(config, "googleapis_commitish"), owlbot_cli_image=__required(config, "owlbot_cli_image"), synthtool_commitish=__required(config, "synthtool_commitish"), - template_excludes=__required(config, 'template_excludes'), + template_excludes=__required(config, "template_excludes"), path_to_yaml=path_to_yaml, libraries=parsed_libraries, ) diff --git a/library_generation/owlbot/src/apply_repo_templates.py b/library_generation/owlbot/src/apply_repo_templates.py index 7b6a463cc6..929ffb66f2 100644 --- a/library_generation/owlbot/src/apply_repo_templates.py +++ b/library_generation/owlbot/src/apply_repo_templates.py @@ -13,31 +13,30 @@ from library_generation.model.generation_config import from_yaml script_dir = os.path.dirname(os.path.realpath(__file__)) -repo_templates_path = os.path.join(script_dir, '..', 'templates', 'java_library') +repo_templates_path = os.path.join(script_dir, "..", "templates", "java_library") + def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None: - config = from_yaml(configuration_yaml_path) - print(f'repo_templates_path: {repo_templates_path}') - print(f'excludes: {config.template_excludes}') - common_templates( - excludes=config.template_excludes, - template_path=Path(repo_templates_path), - monorepo=monorepo - ) + config = from_yaml(configuration_yaml_path) + print(f"repo_templates_path: {repo_templates_path}") + print(f"excludes: {config.template_excludes}") + common_templates( + excludes=config.template_excludes, + template_path=Path(repo_templates_path), + monorepo=monorepo, + ) def main(argv: Sequence[str]) -> None: - if len(argv) != 3: - raise ValueError("Usage: python apply-repo-templates.py configuration_yaml_path monorepo") - - configuration_yaml_path = argv[1] - monorepo = argv[2] - apply_repo_templates( - configuration_yaml_path, - monorepo.lower() == 'true' - ) + if len(argv) != 3: + raise ValueError( + "Usage: python apply-repo-templates.py configuration_yaml_path monorepo" + ) + configuration_yaml_path = argv[1] + monorepo = argv[2] + apply_repo_templates(configuration_yaml_path, monorepo.lower() == "true") if __name__ == "__main__": - main(sys.argv) + main(sys.argv) diff --git a/library_generation/owlbot/src/fix-poms.py b/library_generation/owlbot/src/fix-poms.py index 87df4a46bb..b57012fcf8 100644 --- a/library_generation/owlbot/src/fix-poms.py +++ b/library_generation/owlbot/src/fix-poms.py @@ -285,9 +285,11 @@ def update_bom_pom(filename: str, modules: List[module.Module]): # https://github.com/googleapis/google-cloud-java/issues/9304 def __proto_group_id(main_artifact_group_id: str) -> str: prefix = "com.google" - list_of_group_id = ["com.google.cloud", - "com.google.area120", - "com.google.analytics"] + list_of_group_id = [ + "com.google.cloud", + "com.google.area120", + "com.google.analytics", + ] if main_artifact_group_id not in list_of_group_id: prefix = main_artifact_group_id return f"{prefix}.api.grpc" @@ -343,11 +345,17 @@ def main(versions_file, monorepo): main_module = existing_modules[artifact_id] # Artifact ID is part of distribution name field in .repo-metadata.json - if artifact_id in ["grafeas", "google-cloud-dns", - "google-cloud-notification", "google-iam-policy"]: + if artifact_id in [ + "grafeas", + "google-cloud-dns", + "google-cloud-notification", + "google-iam-policy", + ]: # There are special libraries that are not automatically generated - print(f"Skipping a special case library {artifact_id} that do not have " - " the standard module structure.") + print( + f"Skipping a special case library {artifact_id} that do not have " + " the standard module structure." + ) return parent_artifact_id = f"{artifact_id}-parent" @@ -383,8 +391,10 @@ def main(versions_file, monorepo): version=main_module.version, release_version=main_module.release_version, ) - if path not in excluded_dependencies_list \ - and path not in main_module.artifact_id: + if ( + path not in excluded_dependencies_list + and path not in main_module.artifact_id + ): required_dependencies[path] = module.Module( group_id=__proto_group_id(group_id), artifact_id=path, @@ -401,8 +411,10 @@ def main(versions_file, monorepo): main_module=main_module, monorepo=monorepo, ) - if path not in excluded_dependencies_list \ - and path not in main_module.artifact_id: + if ( + path not in excluded_dependencies_list + and path not in main_module.artifact_id + ): required_dependencies[path] = module.Module( group_id=__proto_group_id(group_id), artifact_id=path, @@ -418,8 +430,10 @@ def main(versions_file, monorepo): version=main_module.version, release_version=main_module.release_version, ) - if path not in excluded_dependencies_list \ - and path not in main_module.artifact_id: + if ( + path not in excluded_dependencies_list + and path not in main_module.artifact_id + ): required_dependencies[path] = module.Module( group_id=__proto_group_id(group_id), artifact_id=path, @@ -439,8 +453,10 @@ def main(versions_file, monorepo): proto_module=existing_modules[proto_artifact_id], monorepo=monorepo, ) - if path not in excluded_dependencies_list \ - and path not in main_module.artifact_id: + if ( + path not in excluded_dependencies_list + and path not in main_module.artifact_id + ): required_dependencies[path] = module.Module( group_id=__proto_group_id(group_id), artifact_id=path, @@ -451,13 +467,13 @@ def main(versions_file, monorepo): module for module in required_dependencies.values() if module.artifact_id.startswith("proto-") - and module.artifact_id not in parent_artifact_id + and module.artifact_id not in parent_artifact_id ] grpc_modules = [ module for module in required_dependencies.values() - if module.artifact_id.startswith("grpc-") \ - and module.artifact_id not in parent_artifact_id + if module.artifact_id.startswith("grpc-") + and module.artifact_id not in parent_artifact_id ] if main_module in grpc_modules or main_module in proto_modules: modules = grpc_modules + proto_modules @@ -489,12 +505,11 @@ def main(versions_file, monorepo): if os.path.isfile(f"{artifact_id}-bom/pom.xml"): print("updating modules in bom pom.xml") - if artifact_id+"-bom" not in excluded_poms_list: + if artifact_id + "-bom" not in excluded_poms_list: update_bom_pom(f"{artifact_id}-bom/pom.xml", modules) - elif artifact_id+"-bom" not in excluded_poms_list: + elif artifact_id + "-bom" not in excluded_poms_list: print("creating missing bom pom.xml") - monorepo_version = __get_monorepo_version(versions_file) \ - if monorepo else "" + monorepo_version = __get_monorepo_version(versions_file) if monorepo else "" templates.render( template_name="bom_pom.xml.j2", output_name=f"{artifact_id}-bom/pom.xml", @@ -503,7 +518,7 @@ def main(versions_file, monorepo): modules=modules, main_module=main_module, monorepo=monorepo, - monorepo_version=monorepo_version + monorepo_version=monorepo_version, ) if os.path.isfile("pom.xml"): @@ -511,8 +526,7 @@ def main(versions_file, monorepo): update_parent_pom("pom.xml", modules) else: print("creating missing parent pom.xml") - monorepo_version = __get_monorepo_version(versions_file) \ - if monorepo else "" + monorepo_version = __get_monorepo_version(versions_file) if monorepo else "" templates.render( template_name="parent_pom.xml.j2", output_name="./pom.xml", @@ -521,7 +535,7 @@ def main(versions_file, monorepo): main_module=main_module, name=name, monorepo=monorepo, - monorepo_version=monorepo_version + monorepo_version=monorepo_version, ) print(f"updating modules in {versions_file}") @@ -537,13 +551,15 @@ def main(versions_file, monorepo): release_version=main_module.release_version, ) templates.render( - template_name="versions.txt.j2", output_name=versions_file, modules=existing_modules.values(), + template_name="versions.txt.j2", + output_name=versions_file, + modules=existing_modules.values(), ) if __name__ == "__main__": versions_file = sys.argv[1] monorepo = sys.argv[2] - if monorepo == 'true': + if monorepo == "true": monorepo = True main(versions_file, monorepo) diff --git a/library_generation/owlbot/src/gen-template.py b/library_generation/owlbot/src/gen-template.py index fd3015ebf8..30f8c43529 100644 --- a/library_generation/owlbot/src/gen-template.py +++ b/library_generation/owlbot/src/gen-template.py @@ -24,7 +24,8 @@ @click.command() @click.option( - "--folder", help="Path to folder of templates", + "--folder", + help="Path to folder of templates", ) @click.option("--file", help="Path to template file") @click.option( @@ -34,7 +35,9 @@ required=True, ) @click.option( - "--output", help="Path to output", default=".", + "--output", + help="Path to output", + default=".", ) def main(folder: str, file: str, data: List[str], output: str): """Generate templates""" diff --git a/library_generation/setup.py b/library_generation/setup.py index 132cee48f0..c8b3418cf4 100644 --- a/library_generation/setup.py +++ b/library_generation/setup.py @@ -4,20 +4,21 @@ from setuptools import setup -setup(name='library_generation', - version='0.1', - package_dir={ - 'library_generation': '.', - }, - package_data={ - 'library_generation': [ - '*.sh', - 'templates/*.j2', - 'gapic-generator-java-wrapper', - 'requirements.*', - 'owlbot/src/requirements.*', - 'owlbot/bin/*.sh', - 'owlbot/templates/**/*.j2', - ], - } +setup( + name="library_generation", + version="0.1", + package_dir={ + "library_generation": ".", + }, + package_data={ + "library_generation": [ + "*.sh", + "templates/*.j2", + "gapic-generator-java-wrapper", + "requirements.*", + "owlbot/src/requirements.*", + "owlbot/bin/*.sh", + "owlbot/templates/**/*.j2", + ], + }, ) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 55f9b88658..2380e15452 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -108,7 +108,7 @@ def test_generate_repo(self): @classmethod def __pull_repo_to(cls, dest: Path, repo: str, committish: str): repo_url = f"{repo_prefix}/{repo}" - print(f'Cloning repository {repo_url}') + print(f"Cloning repository {repo_url}") repo = Repo.clone_from(repo_url, dest) repo.git.checkout(committish) diff --git a/library_generation/test/unit_tests.py b/library_generation/test/unit_tests.py index 540f180eff..268b3ff0c8 100644 --- a/library_generation/test/unit_tests.py +++ b/library_generation/test/unit_tests.py @@ -358,7 +358,7 @@ def __get_a_gen_config(num: int): owlbot_cli_image="", synthtool_commitish="", template_excludes=[], - path_to_yaml='.', + path_to_yaml=".", libraries=libraries, ) From f86635d695b4a2dadae34604e37b29a45a8fd950 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 19:55:29 +0000 Subject: [PATCH 28/30] install lirbary_generation python package via setup.py --- .github/workflows/verify_library_generation.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index e497f286cb..d9c7613ee4 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -42,6 +42,7 @@ jobs: set -ex pushd library_generation pip install -r requirements.in + pip install . popd - name: Run integration tests shell: bash From 02bac2268501481f8c72d8a0b1dced1e3eae99b7 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 20:13:39 +0000 Subject: [PATCH 29/30] include owlbot python sources in python lint check --- .github/workflows/verify_library_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index d9c7613ee4..89cec5ddc9 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -124,4 +124,4 @@ jobs: run: | # exclude generated golden files # exclude owlbot until further refaction - black --check library_generation --exclude "(library_generation/owlbot)|(library_generation/test/resources/goldens)" + black --check library_generation --exclude "(library_generation/test/resources/goldens)" From e5406312f6a5e375e4b98b4c4c8565f6732096c1 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 8 Feb 2024 20:18:58 +0000 Subject: [PATCH 30/30] lint owlbot --- library_generation/owlbot/src/poms/module.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/library_generation/owlbot/src/poms/module.py b/library_generation/owlbot/src/poms/module.py index 3beafd22b0..b4274ea969 100644 --- a/library_generation/owlbot/src/poms/module.py +++ b/library_generation/owlbot/src/poms/module.py @@ -35,7 +35,11 @@ def read_module(pom: str) -> Module: if artifact_id.startswith("google-cloud") else "com.google.api.grpc" ) - return Module(group_id=group_id, artifact_id=artifact_id, version=version,) + return Module( + group_id=group_id, + artifact_id=artifact_id, + version=version, + ) def read_modules(service: str) -> List[Module]: