Clean up and optimize PR builds

- don't allow any var args in the entrypoing scripts; specify all configurations via env vars
- specify which projects to build via ALLUXIO_DOCKER_MVN_PROJECT_LIST, add -am flag to build dependent projects
- specify test prefix via ALLUXIO_DOCKER_MVN_TESTS
- remove ALLUXIO_DOCKER_MVN_RUNTOEND
- remove "libfuse version = 2" permutation from fuse integration tests
  - can't pass in an env var that contains a `=`, which was attempted in the format of `ADDITIONAL_ARGS=property=key` and caused parsing failure
- combine several integration test permutations to reduce overall time spent, since the permutations are finishing much faster than the slowest builds  = unit test and checkstyle builds
			pr-link: #17811
			change-id: cid-fca8ed001d33e49867ebb1732e6a180d6726f6b5

.github/workflows/checkstyle.yml

@@ -34,7 +34,6 @@ jobs:
           mkdir -p ~/.m2
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
           ALLUXIO_CHECKSTYLE=true \
           dev/github/run_docker.sh
         timeout-minutes: 60

.github/workflows/java8_integration_tests.yml

@@ -1,5 +1,11 @@
 name: Java 8 Integration Tests
 
+# This build runs the default integration tests in the dora/tests module
+# It runs all the tests besides those categorized as:
+# - Fault tolerance = alluxio.server.ft
+# - Web UI = alluxio.web
+# - FUSE = alluxio.client.fuse
+
 on: [pull_request]
 
 jobs:
@@ -11,15 +17,11 @@ jobs:
       matrix:
         modules:
           - >-
-            alluxio.client.cli.**,!alluxio.client.cli.fs.**
-          - >-
-            alluxio.client.cli.fs.**
-          - >-
-            alluxio.client.fs.**,!alluxio.client.fs.concurrent.**,!alluxio.client.fs.io.**
+            alluxio.client.cli.**
           - >-
-            alluxio.client.fs.concurrent.**,alluxio.client.fs.io.**
+            alluxio.client.fs.**
           - >-
-            alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**,!alluxio.client.fuse.**
+            alluxio.client.**,!alluxio.client.fs.**,!alluxio.client.cli.**
           - >-
             alluxio.job.**,alluxio.master.**,alluxio.stress.**
           - >-
@@ -56,9 +58,9 @@ jobs:
           mkdir -p ~/.m2
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
-          ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!dora/shaded/client,!dora/integration/tools/hms,!dora/integration/yarn,!assembly/client,!assembly/server,!dora/table/server/underdb/glue,!dora/underfs/hdfs,!dora/underfs/ozone,!dora/underfs/adl,!dora/underfs/abfs,!dora/underfs/cosn,!dora/underfs/wasb,!dora/underfs/cos,!dora/underfs/kodo,!dora/underfs/oss,!dora/underfs/swift \
-          dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl dora/tests
+          ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests \
+          ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \
+          dev/github/run_docker.sh
         timeout-minutes: 60
 
       - name: Archive artifacts

.github/workflows/java8_integration_tests_ft.yml

@@ -1,5 +1,10 @@
 name: Java 8 Fault Tolerant Integration Tests
 
+# This build runs the fault tolerance specific integration tests in the dora/tests module
+# The key differences between this build and the default integration test build are
+# - Only runs integration tests with prefix alluxio.server.ft
+# - Sets fork count to 1 to prevent flakey errors due to running concurrent tests; default count is 2
+
 on: [pull_request]
 
 jobs:
@@ -48,9 +53,9 @@ jobs:
           ALLUXIO_DOCKER_FORK_COUNT=1 \
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
-          ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!dora/shaded/client,!dora/integration/tools/hms,!dora/integration/yarn,!assembly/client,!assembly/server,!dora/table/server/underdb/glue,!dora/underfs/hdfs,!dora/underfs/ozone,!dora/underfs/adl,!dora/underfs/abfs,!dora/underfs/cosn,!dora/underfs/wasb,!dora/underfs/cos,!dora/underfs/kodo,!dora/underfs/oss,!dora/underfs/swift \
-          dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl dora/tests
+          ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests \
+          ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \
+          dev/github/run_docker.sh
         timeout-minutes: 60
 
       - name: Archive artifacts

.github/workflows/java8_integration_tests_webui.yml

@@ -1,5 +1,10 @@
 name: Java 8 Integration Tests (w/ webui)
 
+# This build runs the integration tests in the dora/tests module that require the web UI
+# The key differences between this build and the default integration test build are
+# - Only runs integration tests with prefix alluxio.web
+# - Adds webui to the project list
+
 on: [pull_request]
 
 jobs:
@@ -44,9 +49,9 @@ jobs:
           mkdir -p ~/.m2
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
-          ALLUXIO_DOCKER_MVN_PROJECT_LIST=!dora/shaded/client,!dora/integration/tools/hms,!dora/integration/yarn,!assembly/client,!assembly/server \
-          dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl dora/tests
+          ALLUXIO_DOCKER_MVN_PROJECT_LIST=dora/tests,webui \
+          ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \
+          dev/github/run_docker.sh
         timeout-minutes: 60
 
       - name: Archive artifacts

.github/workflows/java8_unit_tests.yml

@@ -46,9 +46,9 @@ jobs:
           mkdir -p ~/.m2
           ALLUXIO_DOCKER_NO_TTY=true \
           ALLUXIO_DOCKER_GIT_CLEAN=true \
-          ALLUXIO_DOCKER_MVN_RUNTOEND=true \
-          ALLUXIO_DOCKER_MVN_PROJECT_LIST=!webui,!dora/shaded/client,!assembly/client,!assembly/server \
-          dev/github/run_docker.sh "\"-Dtest=${{ matrix.modules }}\"" -pl '!dora/tests,!webui'
+          ALLUXIO_DOCKER_MVN_PROJECT_LIST=\!assembly/client,\!assembly/server,\!dora/shaded/client,\!dora/tests,\!webui \
+          ALLUXIO_DOCKER_MVN_TESTS=${{ matrix.modules }} \
+          dev/github/run_docker.sh
         timeout-minutes: 60
 
       - name: Archive artifacts

dev/github/run_checks.sh

@@ -25,12 +25,9 @@ fi
 export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS"
 
 # Always use java 8 to compile the source code
-JAVA_HOME_BACKUP=${JAVA_HOME}
-PATH_BACKUP=${PATH}
 JAVA_HOME=/usr/local/openjdk-8
 PATH=$JAVA_HOME/bin:$PATH
-mvn -Duser.home=/home/jenkins -T 4C clean install -DskipTests ${mvn_args}
+mvn -Duser.home=/home/jenkins -T 4C clean install -DskipTests
 
 ./dev/scripts/check-docs.sh
 ./dev/scripts/build-artifact.sh ufsVersionCheck
-

dev/github/run_docker.sh

@@ -13,51 +13,38 @@
 set -ex
 
 function main {
-  if [ -z "${ALLUXIO_DOCKER_ID}" ]
-  then
+  if [ -z "${ALLUXIO_DOCKER_ID}" ]; then
     ALLUXIO_DOCKER_ID="$(id -u)"
   fi
-  if [ -z "${ALLUXIO_DOCKER_M2}" ]
-  then
+
+  if [ -z "${ALLUXIO_DOCKER_M2}" ]; then
     ALLUXIO_DOCKER_M2="${HOME}/.m2"
   fi
-  if [ -z "${ALLUXIO_DOCKER_IMAGE}" ]
-  then
+
+  if [ -z "${ALLUXIO_DOCKER_IMAGE}" ]; then
     ALLUXIO_DOCKER_IMAGE="alluxio/alluxio-maven:0.1.3-jdk8"
   fi
 
   local run_args="--rm"
 
-  if [ -z ${ALLUXIO_DOCKER_NO_TTY} ]
-  then
+  if [ -z "${ALLUXIO_DOCKER_NO_TTY}" ]; then
     run_args+=" -it"
   fi
 
-  if [ -n "${ALLUXIO_DOCKER_FORK_COUNT}" ]
-   then
+  if [ -n "${ALLUXIO_DOCKER_FORK_COUNT}" ]; then
      run_args+=" -e ALLUXIO_FORK_COUNT=${ALLUXIO_DOCKER_FORK_COUNT}"
   fi
 
-  if [ -n "${ALLUXIO_DOCKER_GIT_CLEAN}" ]
-  then
+  if [ -n "${ALLUXIO_DOCKER_GIT_CLEAN}" ]; then
     run_args+=" -e ALLUXIO_GIT_CLEAN=true"
   fi
 
-  if [ -n "${ALLUXIO_DOCKER_MVN_RUNTOEND}" ]
-  then
-    run_args+=" -e ALLUXIO_MVN_RUNTOEND=true"
-  fi
-
-  if [ -n "${ALLUXIO_DOCKER_MVN_PROJECT_LIST}" ]
-  then
+  if [ -n "${ALLUXIO_DOCKER_MVN_PROJECT_LIST}" ]; then
     run_args+=" -e ALLUXIO_MVN_PROJECT_LIST=${ALLUXIO_DOCKER_MVN_PROJECT_LIST}"
   fi
 
-  if [ -n "${ALLUXIO_SONAR_ARGS}" ]
-  then
-    # write out to a file, in case there are spaces in the args
-    echo "ALLUXIO_SONAR_ARGS=${ALLUXIO_SONAR_ARGS}" > /tmp/.env
-    run_args+=" --env-file /tmp/.env"
+  if [ -n "${ALLUXIO_DOCKER_MVN_TESTS}" ]; then
+    run_args+=" -e ALLUXIO_MVN_TESTS=${ALLUXIO_DOCKER_MVN_TESTS}"
   fi
 
   local home="/home/jenkins"
@@ -84,14 +71,13 @@ function main {
   run_args+=" -e ALLUXIO_USE_FIXED_TEST_PORTS=true"
   run_args+=" -e ALLUXIO_PORT_COORDINATION_DIR=${home}"
 
-  if [ -n "${ALLUXIO_CHECKSTYLE}" ]
-  then
+  if [ -n "${ALLUXIO_CHECKSTYLE}" ]; then
     run_args+=" --entrypoint=dev/github/run_checks.sh"
   else
     run_args+=" --entrypoint=dev/github/run_tests.sh"
   fi
 
-  docker run ${run_args} ${ALLUXIO_DOCKER_IMAGE} $@
+  docker run ${run_args} ${ALLUXIO_DOCKER_IMAGE}
 }
 
 main "$@"

dev/github/run_tests.sh

@@ -15,28 +15,19 @@
 #
 set -ex
 
-if [ -z "${ALLUXIO_FORK_COUNT}" ]
-then
+if [ -z "${ALLUXIO_FORK_COUNT}" ]; then
   ALLUXIO_FORK_COUNT=2
 fi
 
-if [ -n "${ALLUXIO_GIT_CLEAN}" ]
-then
+if [ -n "${ALLUXIO_GIT_CLEAN}" ]; then
   # https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor
   git config --global --add safe.directory '*'
   git clean -fdx
 fi
 
 mvn_args=""
-if [ -n "${ALLUXIO_MVN_RUNTOEND}" ]
-then
-  mvn_args+=" -fn -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false --fail-at-end"
-fi
-
-mvn_project_list=""
-if [ -n "${ALLUXIO_MVN_PROJECT_LIST}" ]
-then
-  mvn_project_list+=" -pl ${ALLUXIO_MVN_PROJECT_LIST}"
+if [ -n "${ALLUXIO_MVN_PROJECT_LIST}" ]; then
+  mvn_args+="-am -pl ${ALLUXIO_MVN_PROJECT_LIST}"
 fi
 
 export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS"
@@ -54,12 +45,15 @@ myuid=$(id -u)
 mygid=$(id -g)
 echo "$myuid:x:$myuid:$mygid:anonymous uid:/home/jenkins:/bin/false" >> /etc/passwd
 
-export MAVEN_OPTS="-Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS"
-
 # Revert back to the image default java version to run the test
 JAVA_HOME=${JAVA_HOME_BACKUP}
 PATH=${PATH_BACKUP}
 
+mvn_args+=" -fn -DfailIfNoTests=false -Dsurefire.failIfNoSpecifiedTests=false --fail-at-end"
+if [ -n "${ALLUXIO_MVN_TESTS}" ]; then
+  mvn_args+=" -Dtest=${ALLUXIO_MVN_TESTS}"
+fi
+
 # Run tests
 mvn -Duser.home=/home/jenkins test -Dmaven.main.skip -Dskip.protoc=true -Dmaven.javadoc.skip -Dlicense.skip=true \
--Dcheckstyle.skip=true -Dfindbugs.skip=true -Dsurefire.forkCount=${ALLUXIO_FORK_COUNT} ${mvn_args} $@
+-Dcheckstyle.skip=true -Dfindbugs.skip=true -Dsurefire.forkCount=${ALLUXIO_FORK_COUNT} ${mvn_args}