From 3f6718168ac2718f530bfa115b93378601192306 Mon Sep 17 00:00:00 2001 From: Rico Chiu Date: Mon, 24 Jul 2023 16:03:15 -0700 Subject: [PATCH] Clean up and optimize PR builds - don't allow any var args in the entrypoing scripts; specify all configurations via env vars - specify which projects to build via ALLUXIO_DOCKER_MVN_PROJECT_LIST, add -am flag to build dependent projects - specify test prefix via ALLUXIO_DOCKER_MVN_TESTS - remove ALLUXIO_DOCKER_MVN_RUNTOEND - remove "libfuse version = 2" permutation from fuse integration tests - can't pass in an env var that contains a `=`, which was attempted in the format of `ADDITIONAL_ARGS=property=key` and caused parsing failure - combine several integration test permutations to reduce overall time spent, since the permutations are finishing much faster than the slowest builds = unit test and checkstyle builds pr-link: Alluxio/alluxio#17811 change-id: cid-fca8ed001d33e49867ebb1732e6a180d6726f6b5 --- .github/workflows/checkstyle.yml | 1 - .github/workflows/fuse_integration_tests.yml | 61 ------------------- .github/workflows/java8_integration_tests.yml | 22 ++++--- .../workflows/java8_integration_tests_ft.yml | 11 +++- .../java8_integration_tests_webui.yml | 11 +++- .github/workflows/java8_unit_tests.yml | 6 +- dev/github/run_checks.sh | 5 +- dev/github/run_docker.sh | 40 ++++-------- dev/github/run_tests.sh | 26 +++----- 9 files changed, 55 insertions(+), 128 deletions(-) delete mode 100644 .github/workflows/fuse_integration_tests.yml diff --git a/.github/workflows/checkstyle.yml b/.github/workflows/checkstyle.yml index 0764a5d7b0c5..ade0d9a911f5 100644 --- a/.github/workflows/checkstyle.yml +++ b/.github/workflows/checkstyle.yml @@ -34,7 +34,6 @@ jobs: mkdir -p ~/.m2 ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ ALLUXIO_CHECKSTYLE=true \ dev/github/run_docker.sh timeout-minutes: 60 diff --git a/.github/workflows/fuse_integration_tests.yml b/.github/workflows/fuse_integration_tests.yml deleted file mode 100644 index e5ab4606f25f..000000000000 --- a/.github/workflows/fuse_integration_tests.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Fuse Integration Tests - -on: [pull_request] - -jobs: - build: - name: "modules: fuse" - - strategy: - fail-fast: false - matrix: - java: ["11"] - version: ["2", "3"] - - runs-on: ubuntu-latest - if: "!contains(github.event.pull_request.title, 'DOCFIX') && - !contains(github.event.pull_request.title, 'SKIPCI')" - - steps: - - name: checkout repo - uses: actions/checkout@v3 - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: '10.11.0' - - - name: Cache local Maven repository - uses: actions/cache@v3 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-java${{ matrix.java }}-${{ hashFiles('**/pom.xml') }} - - - name: Cache local Go modules - uses: actions/cache@v3 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.mod') }} - - - name: Run tests - id: test0 - run: | - mkdir -p ~/.m2 - ALLUXIO_DOCKER_NO_TTY=true \ - ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!dora/shaded/client,!dora/integration/tools/hms,!dora/integration/yarn,!assembly/client,!assembly/server,!dora/table/server/underdb/glue,!dora/underfs/hdfs,!dora/underfs/ozone,!dora/underfs/adl,!dora/underfs/abfs,!dora/underfs/cosn,!dora/underfs/wasb,!dora/underfs/cos,!dora/underfs/kodo,!dora/underfs/oss,!dora/underfs/swift \ - dev/github/run_docker.sh "\"-Dtest=alluxio.client.fuse.**\"" "\"-Dalluxio.fuse.jnifuse.libfuse.version=${{ matrix.version }}\"" -pl dora/tests - timeout-minutes: 60 - - - name: Archive artifacts - continue-on-error: true - uses: actions/upload-artifact@v3 - if: always() - with: - name: artifact - path: | - **/target/surefire-reports/* - **/target/artifacts/* - **/target/logs/* - retention-days: 7 diff --git a/.github/workflows/java8_integration_tests.yml b/.github/workflows/java8_integration_tests.yml index 00b02c4ffb59..4d2e422fe02d 100644 --- a/.github/workflows/java8_integration_tests.yml +++ b/.github/workflows/java8_integration_tests.yml @@ -1,5 +1,11 @@ name: Java 8 Integration Tests +# This build runs the default integration tests in the dora/tests module +# It runs all the tests besides those categorized as: +# - Fault tolerance = alluxio.server.ft +# - Web UI = alluxio.web +# - FUSE = alluxio.client.fuse + on: [pull_request] jobs: @@ -11,15 +17,11 @@ jobs: matrix: modules: - >- - alluxio.client.cli.**,!alluxio.client.cli.fs.** - - >- - alluxio.client.cli.fs.** - - >- - alluxio.client.fs.**,!alluxio.client.fs.concurrent.**,!alluxio.client.fs.io.** + alluxio.client.cli.** - >- - alluxio.client.fs.concurrent.**,alluxio.client.fs.io.** + alluxio.client.fs.** - >- - alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**,!alluxio.client.fuse.** + alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.** - >- alluxio.job.**,alluxio.master.**,alluxio.stress.** - >- @@ -56,9 +58,9 @@ jobs: mkdir -p ~/.m2 ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!dora/shaded/client,!dora/integration/tools/hms,!dora/integration/yarn,!assembly/client,!assembly/server,!dora/table/server/underdb/glue,!dora/underfs/hdfs,!dora/underfs/ozone,!dora/underfs/adl,!dora/underfs/abfs,!dora/underfs/cosn,!dora/underfs/wasb,!dora/underfs/cos,!dora/underfs/kodo,!dora/underfs/oss,!dora/underfs/swift \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl dora/tests + ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh timeout-minutes: 60 - name: Archive artifacts diff --git a/.github/workflows/java8_integration_tests_ft.yml b/.github/workflows/java8_integration_tests_ft.yml index 6fa0bc35233a..d36bc2602cdb 100644 --- a/.github/workflows/java8_integration_tests_ft.yml +++ b/.github/workflows/java8_integration_tests_ft.yml @@ -1,5 +1,10 @@ name: Java 8 Fault Tolerant Integration Tests +# This build runs the fault tolerance specific integration tests in the dora/tests module +# The key differences between this build and the default integration test build are +# - Only runs integration tests with prefix alluxio.server.ft +# - Sets fork count to 1 to prevent flakey errors due to running concurrent tests; default count is 2 + on: [pull_request] jobs: @@ -48,9 +53,9 @@ jobs: ALLUXIO_DOCKER_FORK_COUNT=1 \ ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!dora/shaded/client,!dora/integration/tools/hms,!dora/integration/yarn,!assembly/client,!assembly/server,!dora/table/server/underdb/glue,!dora/underfs/hdfs,!dora/underfs/ozone,!dora/underfs/adl,!dora/underfs/abfs,!dora/underfs/cosn,!dora/underfs/wasb,!dora/underfs/cos,!dora/underfs/kodo,!dora/underfs/oss,!dora/underfs/swift \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl dora/tests + ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh timeout-minutes: 60 - name: Archive artifacts diff --git a/.github/workflows/java8_integration_tests_webui.yml b/.github/workflows/java8_integration_tests_webui.yml index 31e1a1fca306..e9b36dae67ec 100644 --- a/.github/workflows/java8_integration_tests_webui.yml +++ b/.github/workflows/java8_integration_tests_webui.yml @@ -1,5 +1,10 @@ name: Java 8 Integration Tests (w/ webui) +# This build runs the integration tests in the dora/tests module that require the web UI +# The key differences between this build and the default integration test build are +# - Only runs integration tests with prefix alluxio.web +# - Adds webui to the project list + on: [pull_request] jobs: @@ -44,9 +49,9 @@ jobs: mkdir -p ~/.m2 ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!dora/shaded/client,!dora/integration/tools/hms,!dora/integration/yarn,!assembly/client,!assembly/server \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl dora/tests + ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests,webui \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh timeout-minutes: 60 - name: Archive artifacts diff --git a/.github/workflows/java8_unit_tests.yml b/.github/workflows/java8_unit_tests.yml index fe627a3f4858..7f6adc6221f5 100644 --- a/.github/workflows/java8_unit_tests.yml +++ b/.github/workflows/java8_unit_tests.yml @@ -46,9 +46,9 @@ jobs: mkdir -p ~/.m2 ALLUXIO_DOCKER_NO_TTY=true \ ALLUXIO_DOCKER_GIT_CLEAN=true \ - ALLUXIO_DOCKER_MVN_RUNTOEND=true \ - ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!dora/shaded/client,!assembly/client,!assembly/server \ - dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl '!dora/tests,!webui' + ALLUXIO_DOCKER_MVN_PROJECT_LIST=\!assembly/client,\!assembly/server,\!dora/shaded/client,\!dora/tests,\!webui \ + ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \ + dev/github/run_docker.sh timeout-minutes: 60 - name: Archive artifacts diff --git a/dev/github/run_checks.sh b/dev/github/run_checks.sh index 3f3d6f30801a..2e16aee36577 100755 --- a/dev/github/run_checks.sh +++ b/dev/github/run_checks.sh @@ -25,12 +25,9 @@ fi export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" # Always use java 8 to compile the source code -JAVA_HOME_BACKUP=${JAVA_HOME} -PATH_BACKUP=${PATH} JAVA_HOME=/usr/local/openjdk-8 PATH=$JAVA_HOME/bin:$PATH -mvn -Duser.home=/home/jenkins -T 4C clean install -DskipTests ${mvn_args} +mvn -Duser.home=/home/jenkins -T 4C clean install -DskipTests ./dev/scripts/check-docs.sh ./dev/scripts/build-artifact.sh ufsVersionCheck - diff --git a/dev/github/run_docker.sh b/dev/github/run_docker.sh index 316bd73ba7ea..781c859b5c5c 100755 --- a/dev/github/run_docker.sh +++ b/dev/github/run_docker.sh @@ -13,51 +13,38 @@ set -ex function main { - if [ -z "${ALLUXIO_DOCKER_ID}" ] - then + if [ -z "${ALLUXIO_DOCKER_ID}" ]; then ALLUXIO_DOCKER_ID="$(id -u)" fi - if [ -z "${ALLUXIO_DOCKER_M2}" ] - then + + if [ -z "${ALLUXIO_DOCKER_M2}" ]; then ALLUXIO_DOCKER_M2="${HOME}/.m2" fi - if [ -z "${ALLUXIO_DOCKER_IMAGE}" ] - then + + if [ -z "${ALLUXIO_DOCKER_IMAGE}" ]; then ALLUXIO_DOCKER_IMAGE="alluxio/alluxio-maven:0.1.3-jdk8" fi local run_args="--rm" - if [ -z ${ALLUXIO_DOCKER_NO_TTY} ] - then + if [ -z "${ALLUXIO_DOCKER_NO_TTY}" ]; then run_args+=" -it" fi - if [ -n "${ALLUXIO_DOCKER_FORK_COUNT}" ] - then + if [ -n "${ALLUXIO_DOCKER_FORK_COUNT}" ]; then run_args+=" -e ALLUXIO_FORK_COUNT=${ALLUXIO_DOCKER_FORK_COUNT}" fi - if [ -n "${ALLUXIO_DOCKER_GIT_CLEAN}" ] - then + if [ -n "${ALLUXIO_DOCKER_GIT_CLEAN}" ]; then run_args+=" -e ALLUXIO_GIT_CLEAN=true" fi - if [ -n "${ALLUXIO_DOCKER_MVN_RUNTOEND}" ] - then - run_args+=" -e ALLUXIO_MVN_RUNTOEND=true" - fi - - if [ -n "${ALLUXIO_DOCKER_MVN_PROJECT_LIST}" ] - then + if [ -n "${ALLUXIO_DOCKER_MVN_PROJECT_LIST}" ]; then run_args+=" -e ALLUXIO_MVN_PROJECT_LIST=${ALLUXIO_DOCKER_MVN_PROJECT_LIST}" fi - if [ -n "${ALLUXIO_SONAR_ARGS}" ] - then - # write out to a file, in case there are spaces in the args - echo "ALLUXIO_SONAR_ARGS=${ALLUXIO_SONAR_ARGS}" > /tmp/.env - run_args+=" --env-file /tmp/.env" + if [ -n "${ALLUXIO_DOCKER_MVN_TESTS}" ]; then + run_args+=" -e ALLUXIO_MVN_TESTS=${ALLUXIO_DOCKER_MVN_TESTS}" fi local home="/home/jenkins" @@ -84,14 +71,13 @@ function main { run_args+=" -e ALLUXIO_USE_FIXED_TEST_PORTS=true" run_args+=" -e ALLUXIO_PORT_COORDINATION_DIR=${home}" - if [ -n "${ALLUXIO_CHECKSTYLE}" ] - then + if [ -n "${ALLUXIO_CHECKSTYLE}" ]; then run_args+=" --entrypoint=dev/github/run_checks.sh" else run_args+=" --entrypoint=dev/github/run_tests.sh" fi - docker run ${run_args} ${ALLUXIO_DOCKER_IMAGE} $@ + docker run ${run_args} ${ALLUXIO_DOCKER_IMAGE} } main "$@" diff --git a/dev/github/run_tests.sh b/dev/github/run_tests.sh index 988e4519657d..13642ea0bf3d 100755 --- a/dev/github/run_tests.sh +++ b/dev/github/run_tests.sh @@ -15,28 +15,19 @@ # set -ex -if [ -z "${ALLUXIO_FORK_COUNT}" ] -then +if [ -z "${ALLUXIO_FORK_COUNT}" ]; then ALLUXIO_FORK_COUNT=2 fi -if [ -n "${ALLUXIO_GIT_CLEAN}" ] -then +if [ -n "${ALLUXIO_GIT_CLEAN}" ]; then # https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor git config --global --add safe.directory '*' git clean -fdx fi mvn_args="" -if [ -n "${ALLUXIO_MVN_RUNTOEND}" ] -then - mvn_args+=" -fn -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false --fail-at-end" -fi - -mvn_project_list="" -if [ -n "${ALLUXIO_MVN_PROJECT_LIST}" ] -then - mvn_project_list+=" -pl ${ALLUXIO_MVN_PROJECT_LIST}" +if [ -n "${ALLUXIO_MVN_PROJECT_LIST}" ]; then + mvn_args+="-am -pl ${ALLUXIO_MVN_PROJECT_LIST}" fi export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" @@ -54,12 +45,15 @@ myuid=$(id -u) mygid=$(id -g) echo "$myuid:x:$myuid:$mygid:anonymous uid:/home/jenkins:/bin/false" >> /etc/passwd -export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS" - # Revert back to the image default java version to run the test JAVA_HOME=${JAVA_HOME_BACKUP} PATH=${PATH_BACKUP} +mvn_args+=" -fn -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false --fail-at-end" +if [ -n "${ALLUXIO_MVN_TESTS}" ]; then + mvn_args+=" -Dtest=${ALLUXIO_MVN_TESTS}" +fi + # Run tests mvn -Duser.home=/home/jenkins test -Dmaven.main.skip -Dskip.protoc=true -Dmaven.javadoc.skip -Dlicense.skip=true \ --Dcheckstyle.skip=true -Dfindbugs.skip=true -Dsurefire.forkCount=${ALLUXIO_FORK_COUNT} ${mvn_args} $@ +-Dcheckstyle.skip=true -Dfindbugs.skip=true -Dsurefire.forkCount=${ALLUXIO_FORK_COUNT} ${mvn_args}