diff --git a/.github/workflows/check-new-library-versions-in-batch.yml b/.github/workflows/check-new-library-versions-in-batch.yml new file mode 100644 index 000000000..9a5a808f7 --- /dev/null +++ b/.github/workflows/check-new-library-versions-in-batch.yml @@ -0,0 +1,203 @@ +name: "Check new libraries versions" + +on: + schedule: + - cron: '0 0 * * 7' + workflow_dispatch: + +permissions: + contents: write + actions: write + +concurrency: + group: "workflow=${{ github.workflow }},ref=${{ github.event.ref }},pr=${{ github.event.pull_request.id }}" + cancel-in-progress: true + +jobs: + get-all-libraries: + name: "πŸ“‹ Get list of all supported libraries with newer versions" + if: github.repository == 'oracle/graalvm-reachability-metadata' + runs-on: ubuntu-22.04 + timeout-minutes: 5 + permissions: write-all + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + branch: ${{ steps.set-branch-name.outputs.branch }} + steps: + - name: "☁️ Checkout repository" + uses: actions/checkout@v4 + + - name: "πŸ”§ Prepare environment" + uses: graalvm/setup-graalvm@v1 + with: + java-version: '21' + distribution: 'graalvm' + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: "πŸ“… Set branch name" + id: set-branch-name + run: | + BRANCH_NAME="check-new-library-versions/$(date '+%Y-%m-%d')" + echo "branch=$BRANCH_NAME" >> $GITHUB_OUTPUT + + - name: "πŸ•ΈοΈ Populate matrix" + id: set-matrix + run: | + echo "matrix=$(./gradlew fetchExistingLibrariesWithNewerVersions --quiet | sed -n '/\[/,$p')" >> $GITHUB_OUTPUT + + - name: "πŸ”¨ Create branch" + run: | + git config --local user.email "actions@github.com" + git config --local user.name "Github Actions" + git checkout -b "${{ steps.set-branch-name.outputs.branch }}" + git push origin "${{ steps.set-branch-name.outputs.branch }}" + + test-all-metadata: + name: "πŸ§ͺ ${{ matrix.item.name }}" + runs-on: ubuntu-22.04 + needs: get-all-libraries + permissions: write-all + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + strategy: + fail-fast: false + matrix: + item: ${{ fromJson(needs.get-all-libraries.outputs.matrix) }} + steps: + - name: "☁️ Checkout repository" + uses: actions/checkout@v4 + + - name: "πŸ”§ Setup java" + uses: actions/setup-java@v4 + with: + distribution: 'oracle' + java-version: '21' + + - name: "πŸ”§ Prepare environment" + uses: graalvm/setup-graalvm@v1 + with: + set-java-home: 'false' + java-version: 21 + distribution: 'graalvm' + github-token: ${{ secrets.GITHUB_TOKEN }} + native-image-job-reports: 'true' + + - name: "Check for existing failure issue and skip if found" + id: check_existing_issue + run: | + GROUP_ID="$(echo "${{ matrix.item.name }}" | cut -d: -f1)" + ARTIFACT_ID="$(echo "${{ matrix.item.name }}" | cut -d: -f2)" + + readarray -t VERSIONS < <(echo '${{ toJson(matrix.item.versions) }}' | jq -r '.[]') + FIRST_TESTING_VERSION="${VERSIONS[0]}" + + TITLE="Failure detected for $GROUP_ID:$ARTIFACT_ID" + BODY="First failing version $GROUP_ID:$ARTIFACT_ID:$FIRST_TESTING_VERSION" + + ISSUE_NUMBER=$(gh issue list --repo "${{ github.repository }}" --state open --search "$TITLE" --json number,title,body --jq \ + '.[] | select(.title == "'"$TITLE"'") | select(.body == "'"$BODY"'") | .number') + + if [[ -n "$ISSUE_NUMBER" ]]; then + echo "There is no progress since last time this version was tested. Skipping further steps." + exit 0 + fi + + - name: "Extract test path and library version" + id: extract-params + run: | + LIBRARY_PATH=$(echo "${{ matrix.item.name }}" | sed 's/:/\//g') + LATEST_VERSION=$(find tests/src/$LIBRARY_PATH/* -maxdepth 1 -type d | sort -V | tail -1 | cut -d '/' -f5) + TEST_PATH="$LIBRARY_PATH/$LATEST_VERSION" + TEST_COORDINATES=$(echo "$TEST_PATH" | tr / :) + + echo "LATEST_VERSION=$LATEST_VERSION" >> ${GITHUB_ENV} + echo "TEST_PATH=$TEST_PATH" >> ${GITHUB_ENV} + echo "TEST_COORDINATES=$TEST_COORDINATES" >> ${GITHUB_ENV} + + - name: "Pull allowed docker images" + run: ./gradlew pullAllowedDockerImages --coordinates="${{ env.TEST_COORDINATES }}" + + - name: "Disable docker" + run: | + sudo apt-get install openbsd-inetd + sudo bash -c "cat ./.github/workflows/discard-port.conf >> /etc/inetd.conf" + sudo systemctl start inetd + sudo mkdir /etc/systemd/system/docker.service.d + sudo bash -c "cat ./.github/workflows/dockerd.service > /etc/systemd/system/docker.service.d/http-proxy.conf" + sudo systemctl daemon-reload + sudo systemctl restart docker + + - name: "πŸ§ͺ Run '${{ env.TEST_COORDINATES }}' tests" + id: runtests + run: | + bash ./.github/workflows/run-consecutive-tests.sh "${{ env.TEST_COORDINATES }}" '${{ toJson(matrix.item.versions) }}' 2>&1 | tee test_results.txt || true + + # Extract successful versions + grep "^PASSED:" test_results.txt | sed 's/PASSED://g' > successful_versions.txt + echo "successful_versions<> $GITHUB_OUTPUT + cat successful_versions.txt >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # Extract failed version + FAILED_VERSION=$(grep "^FAILED:" test_results.txt | sed 's/FAILED://g') + echo "failed_version=$FAILED_VERSION" >> $GITHUB_OUTPUT + + - name: "βœ”οΈ New library is supported" + if: steps.runtests.outputs.successful_versions != '' + run: | + git config --local user.email "actions@github.com" + git config --local user.name "Github Actions" + git fetch origin ${{ needs.get-all-libraries.outputs.branch }} + git checkout ${{ needs.get-all-libraries.outputs.branch }} + + while read version; do + if [ -n "$version" ]; then + ./gradlew addTestedVersion --coordinates="${{ matrix.item.name }}:$version" --lastSupportedVersion="${{ env.LATEST_VERSION }}" + fi + done < successful_versions.txt + + git add -u + git commit -m "Update tested versions for ${{ matrix.item.name }}" + git push origin ${{ needs.get-all-libraries.outputs.branch }} + + - name: "❗ New library is not supported" + if: steps.runtests.outputs.failed_version != '' + run: | + git config --local user.email "actions@github.com" + git config --local user.name "Github Actions" + + FAILED_VERSION="${{ steps.runtests.outputs.failed_version }}" + REPO="${{ github.repository }}" + TITLE="Failure detected for ${{ matrix.item.name }}" + BODY="First failing version ${{ matrix.item.name }}:$FAILED_VERSION" + + ISSUE_NUMBER=$(gh issue list --repo "$REPO" --state open --search "$TITLE" --json number,title --jq '.[] | select(.title == "'"$TITLE"'") | .number') + if [ -n "$ISSUE_NUMBER" ]; then + echo "Updating existing issue #$ISSUE_NUMBER" + gh issue edit "$ISSUE_NUMBER" --repo "$REPO" --body "$BODY" + else + echo "Creating new issue" + gh issue create --repo "$REPO" --title "$TITLE" --body "$BODY" + fi + + process-results: + name: "πŸ§ͺ Process results" + runs-on: ubuntu-22.04 + permissions: write-all + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + if: ${{ always() }} + needs: + - get-all-libraries + - test-all-metadata + steps: + - name: "☁️ Checkout repository" + uses: actions/checkout@v4 + + - name: "✏️ PR for supported versions" + run: | + git config --local user.email "actions@github.com" + git config --local user.name "Github Actions" + git fetch origin ${{ needs.get-all-libraries.outputs.branch }} + git checkout ${{ needs.get-all-libraries.outputs.branch }} + gh pr create --title "Update supported library versions" --body "This pull request updates supported versions of the existing libraries in the repo" diff --git a/.github/workflows/check-new-library-versions.yml b/.github/workflows/check-new-library-versions.yml deleted file mode 100644 index e012911d4..000000000 --- a/.github/workflows/check-new-library-versions.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: "Check new library versions" - -# The workflow runs bi-weekly alternating with the scheduled release workflow. This way we have enough time to provide metadata for failing tests. -# In case we need more scans, there is a possibility to trigger the workflow manually. -on: - schedule: - - cron: "0 0 8 * *" - - cron: "0 0 22 * *" - workflow_dispatch: - -permissions: - contents: write - actions: write - -concurrency: - group: "workflow = ${{ github.workflow }}, ref = ${{ github.event.ref }}, pr = ${{ github.event.pull_request.id }}" - cancel-in-progress: true - -jobs: - get-all-libraries: - if: github.repository == 'oracle/graalvm-reachability-metadata' - name: "πŸ“‹ Get list of all supported libraries with newer versions" - permissions: write-all - runs-on: "ubuntu-22.04" - timeout-minutes: 5 - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: "☁️ Checkout repository" - uses: actions/checkout@v4 - - name: "πŸ”§ Prepare environment" - uses: graalvm/setup-graalvm@v1 - with: - java-version: '21' - distribution: 'graalvm' - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: "πŸ•ΈοΈ Populate matrix" - id: set-matrix - run: | - ./gradlew fetchExistingLibrariesWithNewerVersions --matrixLimit=200 - - name: "πŸ”¨ Create branch" - run: | - git config --local user.email "actions@github.com" - git config --local user.name "Github Actions" - git switch -C check-new-library-versions/$(date '+%Y-%m-%d') - git push origin check-new-library-versions/$(date '+%Y-%m-%d') - - test-all-metadata: - name: "πŸ§ͺ ${{ matrix.coordinates }} (GraalVM for JDK ${{ matrix.version }} @ ${{ matrix.os }})" - permissions: write-all - runs-on: ${{ matrix.os }} - timeout-minutes: 20 - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: get-all-libraries - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get-all-libraries.outputs.matrix) }} - steps: - - name: "☁️ Checkout repository" - uses: actions/checkout@v4 - - name: "πŸ”§ Setup java" - uses: actions/setup-java@v4 - with: - distribution: 'oracle' - java-version: '21' - - name: "πŸ”§ Prepare environment" - uses: graalvm/setup-graalvm@v1 - with: - set-java-home: 'false' - java-version: ${{ matrix.version }} - distribution: 'graalvm' - github-token: ${{ secrets.GITHUB_TOKEN }} - native-image-job-reports: 'true' - - name: "Extract test path and library version" - run: | - LIBRARY_PATH=$(echo ${{ matrix.coordinates }} | cut -d ':' -f1-2 | sed 's/:/\//g') - LATEST_VERSION=$(find tests/src/$LIBRARY_PATH/* -maxdepth 1 -type d | sort -V | tail -1 | cut -d '/' -f5) - TEST_PATH="$LIBRARY_PATH/$LATEST_VERSION" - TEST_COORDINATES=$(echo "$TEST_PATH" | tr / :) - - echo "LATEST_VERSION=$LATEST_VERSION" >> ${GITHUB_ENV} - echo "TEST_PATH=$TEST_PATH" >> ${GITHUB_ENV} - echo "TEST_COORDINATES=$TEST_COORDINATES" >> ${GITHUB_ENV} - - name: "Pull allowed docker images" - run: | - ./gradlew pullAllowedDockerImages --coordinates=${{ env.TEST_COORDINATES }} - - name: "Disable docker" - run: | - sudo apt-get install openbsd-inetd - sudo bash -c "cat ./.github/workflows/discard-port.conf >> /etc/inetd.conf" - sudo systemctl start inetd - sudo mkdir /etc/systemd/system/docker.service.d - sudo bash -c "cat ./.github/workflows/dockerd.service > /etc/systemd/system/docker.service.d/http-proxy.conf" - sudo systemctl daemon-reload - sudo systemctl restart docker - - name: "πŸ§ͺ Run '${{ env.TEST_COORDINATES }}' tests" - run: | - TESTING_VERSION=$(echo ${{ matrix.coordinates }} | cut -d ":" -f3) - export GVM_TCK_LV=$TESTING_VERSION - - ./gradlew test -Pcoordinates=${{ env.TEST_COORDINATES }} - - name: "βœ”οΈ New library is supported" - if: success() - run: | - bash ./.github/workflows/tryPushVersionsUpdate.sh ${{ matrix.coordinates }} ${{ env.LATEST_VERSION }} - - name: "❗ New library is not supported" - if: failure() - run: | - LIB=$(echo "${{ matrix.coordinates }}" | sed 's/:/_/g') - touch $LIB - echo "UNSUPPORTED_LIB=$LIB" >> $GITHUB_ENV - - name: "Upload artifacts" - if: failure() - id: upload - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: ${{ env.UNSUPPORTED_LIB }} - path: ${{ env.UNSUPPORTED_LIB }} - retention-days: 1 - - process-results: - name: "πŸ§ͺ Process results" - runs-on: "ubuntu-22.04" - if: ${{ always() }} - needs: - - get-all-libraries - - test-all-metadata - permissions: write-all - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: "☁️ Checkout repository" - uses: actions/checkout@v4 - - name: "πŸ”§ Setup java" - uses: actions/setup-java@v4 - with: - distribution: 'oracle' - java-version: '21' - - name: "✏️ PR for supported versions" - run: | - git config --local user.email "actions@github.com" - git config --local user.name "Github Actions" - git fetch origin check-new-library-versions/$(date '+%Y-%m-%d') - git checkout check-new-library-versions/$(date '+%Y-%m-%d') - gh pr create --title "Update supported library versions" --body "This pull request updates supported versions of the existing libraries in the repo" - - name: "Download artifacts for unsupported versions" - uses: actions/download-artifact@v4 - with: - path: ./unsupported - - name: "✏️ Issue for unsupported versions" - run: | - git config --local user.email "actions@github.com" - git config --local user.name "Github Actions" - - LABEL="library-update" - ALL_LIBRARIES=$(ls unsupported) - FORMATTED_BODY=$(./gradlew -q groupLibrariesByName --libraries="$ALL_LIBRARIES") - - EXISTING_ISSUE=$(gh issue list --label "$LABEL" --state open --limit 1 --json url | jq -r '.[0].url') - if [ $EXISTING_ISSUE != "null" ]; then - gh issue edit $EXISTING_ISSUE --body "$FORMATTED_BODY" - else - gh issue create --title "List unsupported libraries versions" --body "$FORMATTED_BODY" --label $LABEL - fi diff --git a/.github/workflows/run-consecutive-tests.sh b/.github/workflows/run-consecutive-tests.sh new file mode 100644 index 000000000..d1fc194d2 --- /dev/null +++ b/.github/workflows/run-consecutive-tests.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -u +set -x + +if [ $# -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Input parameters +TEST_COORDINATES="$1" +VERSIONS_JSON="$2" + +# Remove surrounding single quotes if present (when called from workflow) +VERSIONS_JSON="${VERSIONS_JSON#"${VERSIONS_JSON%%[!\']*}"}" +VERSIONS_JSON="${VERSIONS_JSON%"${VERSIONS_JSON##*[!\']}"}" + +# Parse versions with jq +readarray -t VERSIONS < <(echo "$VERSIONS_JSON" | jq -r '.[]') + +for VERSION in "${VERSIONS[@]}"; do + echo "Running test with GVM_TCK_LV=$VERSION and coordinates=$TEST_COORDINATES" + GVM_TCK_LV="$VERSION" ./gradlew test -Pcoordinates="$TEST_COORDINATES" + RESULT=$? + + ATTEMPTS=1 + # maybe we failed because the test was flaky => try two more times to be sure + while [ "$RESULT" -ne 0 ] && [ $ATTEMPTS -le 2 ]; do + echo "Re-running the test with GVM_TCK_LV=$VERSION and coordinates=$TEST_COORDINATES" + GVM_TCK_LV="$VERSION" ./gradlew clean test -Pcoordinates="$TEST_COORDINATES" + RESULT=$? + ATTEMPTS=$((ATTEMPTS + 1)) + done + + if [ "$RESULT" -eq 0 ]; then + echo "PASSED:$VERSION" + else + echo "FAILED:$VERSION" + break + fi +done + +exit 0 diff --git a/.github/workflows/tryPushVersionsUpdate.sh b/.github/workflows/tryPushVersionsUpdate.sh deleted file mode 100644 index 4d807c3bd..000000000 --- a/.github/workflows/tryPushVersionsUpdate.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash -:' This script tries to run addTestedVersion gradle task which adds new version in the tested-versions list of the proper index.json file. -Since the script could be executed from multiple parallel jobs, we want to avoid two things here: overwriting of previous changes and merge conflicts. -To prevent overwriting of changes that some job already created, we only push changes from the current job if we are 0 commits behind the origin branch. -Once that is achieved, we can try to push changes. -If the push was rejected because of a merge conflict, we are: removing changes of the current job, rebasing, and doing the process again until it succeeds. -' - -set -x - -git config --local user.email "actions@github.com" -git config --local user.name "Github Actions" - -BRANCH="check-new-library-versions/$(date '+%Y-%m-%d')" -git fetch origin "$BRANCH" -git checkout "$BRANCH" - -while [ true ] -do - # update the list of tested versions - ./gradlew addTestedVersion --coordinates="$1" --lastSupportedVersion="$2" - - # commit changes - git add -u - git commit -m "$1" - - # only push changes if we are not behind the remote branch - if [ "$(git rev-list --count origin/$BRANCH --not $BRANCH)" -eq 0 ] - then - # try to push changes - git push origin "$BRANCH" - PUSH_RETVAL=$? - if [ "$PUSH_RETVAL" -eq 0 ] - then - # if the push was successful, we can exit the loop - break - fi - fi - - # we are either behind the remote branch or we have a merge conflict => remove changes and rebase accepting incoming changes - git reset --hard HEAD~1 - git fetch origin "$BRANCH" - git rebase -X theirs "origin/$BRANCH" -done - - diff --git a/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/updaters/FetchExistingLibrariesWithNewerVersionsTask.groovy b/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/updaters/FetchExistingLibrariesWithNewerVersionsTask.groovy index 23197c1b7..c230a5693 100644 --- a/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/updaters/FetchExistingLibrariesWithNewerVersionsTask.groovy +++ b/tests/tck-build-logic/src/main/groovy/org/graalvm/internal/tck/updaters/FetchExistingLibrariesWithNewerVersionsTask.groovy @@ -24,10 +24,6 @@ abstract class FetchExistingLibrariesWithNewerVersionsTask extends DefaultTask { @Input abstract ListProperty getAllLibraryCoordinates() - @Input - @Option(option = "matrixLimit", description = "Sets the maximum number of coordinates in the final matrix") - abstract Property getMatrixLimit() - private static final List INFRASTRUCTURE_TESTS = List.of("samples", "org.example") @TaskAction @@ -50,17 +46,15 @@ abstract class FetchExistingLibrariesWithNewerVersionsTask extends DefaultTask { } } - if (newerVersions.size() > getMatrixLimit().get()) { - newerVersions = newerVersions.subList(0, getMatrixLimit().get()) + def map = [:] + newerVersions.each { coord -> + def (group, artifact, version) = coord.tokenize(':') + def key = "${group}:${artifact}" + map[key] = (map[key] ?: []) + version } + def pairs = map.collect { k, v -> [name: k, versions: v] } - def matrix = [ - "coordinates": newerVersions, - "version" : ["17"], - "os" : ["ubuntu-latest"] - ] - - new File(System.getenv("GITHUB_OUTPUT")).append("matrix::${JsonOutput.toJson(matrix)}") + println JsonOutput.toJson(pairs) } static List getNewerVersionsFor(String library, String startingVersion) { @@ -88,14 +82,10 @@ abstract class FetchExistingLibrariesWithNewerVersionsTask extends DefaultTask { int indexOfStartingVersion = allVersions.indexOf(startingVersion); if (indexOfStartingVersion < 0) { - System.out.println("Cannot find starting version in index file: " + libraryName + " for version " + startingVersion); return new ArrayList<>(); } allVersions = allVersions.subList(indexOfStartingVersion, allVersions.size()); - if (allVersions.size() <= 1) { - System.out.println("Cannot find newer versions for " + libraryName + " after the version " + startingVersion); - } return allVersions.subList(1, allVersions.size()); }