-
Notifications
You must be signed in to change notification settings - Fork 53
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ci: add downstream compatibility checks (#1909)
* ci: add downstream compatibility checks * ci: add downstream compatibility github actions job * ci: install xmllint * ci: install xmllint * ci: install xmllint * ci: provide default REPOS_UNDER_TEST value * ci: exit script in google-cloud-java directory * ci: split downstream compatibility into parallel tests * ci: reduce log noise from sdk-platform-java install step * ci: don't fail fast in downstream testing * ci: perform downstream testing on last release, not HEAD * chore: improve comments * ci: add tests for helper scripts
- Loading branch information
1 parent
a18fe80
commit f86eca2
Showing
5 changed files
with
276 additions
and
41 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -270,7 +270,7 @@ jobs: | |
| mvn clirr:check -B -ntp -Dclirr.skip=false -DcomparisonVersion=$SHOWCASE_CLIENT_VERSION | ||
| gapic-generator-java-bom: | ||
| runs-on: ubuntu-latest | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/setup-java@v3 | ||
|
|
@@ -285,3 +285,39 @@ jobs: | |
| uses: googleapis/java-cloud-bom/tests/[email protected] | ||
| with: | ||
| bom-path: gapic-generator-java-bom/pom.xml | ||
|
|
||
| downstream-compatibility: | ||
| runs-on: ubuntu-22.04 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| repo: | ||
| - google-cloud-java | ||
| - java-bigtable | ||
| - java-bigquery | ||
| - java-bigquerystorage | ||
| - java-datastore | ||
| - java-firestore | ||
| - java-logging | ||
| - java-logging-logback | ||
| - java-pubsub | ||
| - java-pubsublite | ||
| - java-spanner-jdbc | ||
| - java-spanner | ||
| - java-storage | ||
| - java-storage-nio | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/setup-java@v3 | ||
| with: | ||
| java-version: 17 | ||
| distribution: temurin | ||
| - run: mvn -version | ||
| - name: Install xmllint | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get -y install libxml2-utils | ||
| - name: Test helper scripts | ||
| run: ./.kokoro/presubmit/common_test.sh | ||
| - name: Perform downstream compatibility testing | ||
| run: REPOS_UNDER_TEST="${{ matrix.repo }}" ./.kokoro/presubmit/downstream-compatibility.sh | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| #!/bin/bash | ||
| # Copyright 2023 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # In the given directory ($1), | ||
| # update the pom.xml's dependency on the given artifact ($2) to the given version ($3) | ||
| # ex: update_dependency google-cloud-java/google-cloud-jar-parent google-cloud-shared-dependencies 1.2.3 | ||
| function update_pom_dependency { | ||
| pushd "$1" || exit 1 | ||
| xmllint --shell pom.xml &>/dev/null <<EOF | ||
| setns x=http://maven.apache.org/POM/4.0.0 | ||
| cd .//x:artifactId[text()="$2"] | ||
| cd ../x:version | ||
| set $3 | ||
| save pom.xml | ||
| EOF | ||
| popd || exit 1 | ||
| } | ||
|
|
||
| # Find all pom.xml files that declare a specific version for the given artifact ($1) | ||
| function find_all_poms_with_versioned_dependency { | ||
| poms=($(find . -name pom.xml)) | ||
| for pom in "${poms[@]}"; do | ||
| if xmllint --xpath "//*[local-name()='artifactId' and text()='$1']/following-sibling::*[local-name()='version']" "$pom" &>/dev/null; then | ||
| found+=("$pom") | ||
| fi | ||
| done | ||
| POMS=(${found[@]}) | ||
| unset found | ||
| export POMS | ||
| } | ||
|
|
||
| # In the given directory ($1), | ||
| # find and update all pom.xmls' dependencies on the given artifact ($2) to the given version ($3) | ||
| # ex: update_all_poms_dependency google-cloud-java google-cloud-shared-dependencies 1.2.3 | ||
| function update_all_poms_dependency { | ||
| pushd "$1" || exit 1 | ||
| find_all_poms_with_versioned_dependency "$2" | ||
| for pom in $POMS; do | ||
| update_pom_dependency "$(dirname "$pom")" "$2" "$3" | ||
| done | ||
| git diff | ||
| popd || exit 1 | ||
| } | ||
|
|
||
| # Parse the version of the pom.xml file in the given directory ($1) | ||
| # ex: VERSION=$(parse_pom_version java-shared-dependencies) | ||
| function parse_pom_version { | ||
| # Namespace (xmlns) prevents xmllint from specifying tag names in XPath | ||
| result=$(sed -e 's/xmlns=".*"//' "$1/pom.xml" | xmllint --xpath '/project/version/text()' -) | ||
|
|
||
| if [ -z "${result}" ]; then | ||
| echo "Version is not found in $1" | ||
| exit 1 | ||
| fi | ||
| echo "$result" | ||
| } | ||
|
|
||
| # ex: find_last_release_version java-bigtable | ||
| # ex: find_last_release_version java-storage 2.22.x | ||
| function find_last_release_version { | ||
| branch=${2-"main"} # Default to using main branch | ||
| curl -s -o "versions_$1.txt" "https://raw.githubusercontent.com/googleapis/$1/$branch/versions.txt" | ||
|
|
||
| # First check to see if there's an entry for the overall repo. Used for google-cloud-java. | ||
| primary_artifact=$(grep -E "^$1" "versions_$1.txt" | head -n 1) | ||
| if [ -z "$primary_artifact" ]; then | ||
| # Otherwise, use the first google-cloud-* artifact's version. | ||
| primary_artifact=$(grep -E "^google-cloud-" "versions_$1.txt" | head -n 1) | ||
| fi | ||
| if [ -z "$primary_artifact" ]; then | ||
| echo "Unable to identify primary artifact for $1" | ||
| exit 1 | ||
| fi | ||
|
|
||
| parts=($(echo "$primary_artifact" | tr ":" "\n")) | ||
| echo "${parts[1]}" | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| #!/bin/bash | ||
| # Copyright 2023 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") | ||
| cd "${scriptDir}/../.." # cd to the root of this repo | ||
| source "$scriptDir/common.sh" | ||
| mkdir -p target | ||
| cd target | ||
|
|
||
| function test_find_all_poms_with_versioned_dependency { | ||
| mkdir -p test_find_all_poms_with_dependency | ||
| pushd test_find_all_poms_with_dependency | ||
| cp ../../showcase/gapic-showcase/pom.xml pom.xml | ||
|
|
||
| find_all_poms_with_versioned_dependency 'truth' | ||
| if [ "${#POMS[@]}" != 1 ]; then | ||
| echo 'find_all_poms_with_versioned_dependency did not find the expected pom' | ||
| exit 1 | ||
| elif [ "${POMS[0]}" != './pom.xml' ]; then | ||
| echo "find_all_poms_with_versioned_dependency found ${POMS[0]} instead of expected ./pom.xml" | ||
| exit 1 | ||
| fi | ||
|
|
||
| find_all_poms_with_versioned_dependency 'gax-grpc' # Versioned by shared-deps | ||
| if [ "${#POMS[@]}" != 0 ]; then | ||
| echo 'find_all_poms_with_versioned_dependency found unexpected pom' | ||
| exit 1 | ||
| fi | ||
|
|
||
| popd | ||
| } | ||
|
|
||
| function test_update_pom_dependency { | ||
| mkdir -p test_update_pom_dependency | ||
| pushd test_update_pom_dependency | ||
| cp ../../showcase/gapic-showcase/pom.xml pom.xml | ||
|
|
||
| update_pom_dependency . truth "99.88.77" | ||
|
|
||
| xmllint --shell pom.xml &>/dev/null <<EOF | ||
| setns x=http://maven.apache.org/POM/4.0.0 | ||
| cd .//x:artifactId[text()="truth"] | ||
| cd ../x:version | ||
| write found-version.txt | ||
| EOF | ||
| if ! grep 99.88.77 found-version.txt &>/dev/null; then | ||
| echo "update_pom_dependency failed to change version to expected value." | ||
| exit 1 | ||
| fi | ||
| rm found-version.txt | ||
| popd | ||
| } | ||
|
|
||
| function test_parse_pom_version { | ||
| mkdir -p test_parse_pom_version | ||
| pushd test_parse_pom_version | ||
| cp ../../showcase/gapic-showcase/pom.xml pom.xml | ||
|
|
||
| VERSION=$(parse_pom_version .) | ||
| if [ "$VERSION" != "0.0.1-SNAPSHOT" ]; then | ||
| echo "parse_pom_version failed to read expected version of gapic-showcase." | ||
| fi | ||
| popd | ||
| } | ||
|
|
||
| test_find_all_poms_with_versioned_dependency | ||
| test_update_pom_dependency | ||
| test_parse_pom_version |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| #!/bin/bash | ||
| # Copyright 2023 Google LLC | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| set -eo pipefail | ||
|
|
||
| # Comma-delimited list of repos to test with the local java-shared-dependencies | ||
| if [ -z "${REPOS_UNDER_TEST}" ]; then | ||
| echo "REPOS_UNDER_TEST must be set to run downstream-compatibility.sh" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Get the directory of the build script | ||
| scriptDir=$(realpath "$(dirname "${BASH_SOURCE[0]}")") | ||
| cd "${scriptDir}/../.." # cd to the root of this repo | ||
| source "$scriptDir/common.sh" | ||
|
|
||
| echo "Setup maven mirror" | ||
| mkdir -p "${HOME}/.m2" | ||
| cp settings.xml "${HOME}/.m2" | ||
|
|
||
| echo "Installing this repo's modules to local maven." | ||
| mvn -q -B -ntp install --projects '!gapic-generator-java' \ | ||
| -Dcheckstyle.skip -Dfmt.skip -DskipTests -T 1C | ||
| SHARED_DEPS_VERSION=$(parse_pom_version java-shared-dependencies) | ||
| echo "Install complete. java-shared-dependencies = $SHARED_DEPS_VERSION" | ||
|
|
||
| pushd java-shared-dependencies/target | ||
| for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma | ||
| # Perform testing on last release, not HEAD | ||
| last_release=$(find_last_release_version "$repo") | ||
| git clone "https://github.com/googleapis/$repo.git" --depth=1 --branch "v$last_release" | ||
| update_all_poms_dependency "$repo" google-cloud-shared-dependencies "$SHARED_DEPS_VERSION" | ||
| pushd "$repo" | ||
| JOB_TYPE="test" ./.kokoro/build.sh | ||
| popd | ||
| done | ||
| popd |