[HUDI-6300] Fix file size parallelism not work when init metadata table #26651
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Java CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - 'release-*' | |
| pull_request: | |
| paths-ignore: | |
| - '**.bmp' | |
| - '**.gif' | |
| - '**.jpg' | |
| - '**.jpeg' | |
| - '**.md' | |
| - '**.pdf' | |
| - '**.png' | |
| - '**.svg' | |
| - '**.yaml' | |
| - '**.yml' | |
| - '.gitignore' | |
| branches: | |
| - master | |
| - 'release-*' | |
| env: | |
| MVN_ARGS: -e -ntp -B -V -Dgpg.skip -Djacoco.skip -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn | |
| SPARK_COMMON_MODULES: hudi-spark-datasource/hudi-spark,hudi-spark-datasource/hudi-spark-common | |
| jobs: | |
| validate-source: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Check Binary Files | |
| run: ./scripts/release/validate_source_binary_files.sh | |
| - name: Check Copyright | |
| run: | | |
| ./scripts/release/create_source_directory.sh hudi-tmp-repo | |
| cd hudi-tmp-repo | |
| ./scripts/release/validate_source_copyright.sh | |
| - name: RAT check | |
| run: ./scripts/release/validate_source_rat.sh | |
| test-spark: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - scalaProfile: "scala-2.11" | |
| sparkProfile: "spark2.4" | |
| sparkModules: "hudi-spark-datasource/hudi-spark2" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark3.0" | |
| sparkModules: "hudi-spark-datasource/hudi-spark3.0.x" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark3.1" | |
| sparkModules: "hudi-spark-datasource/hudi-spark3.1.x" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark3.2" | |
| sparkModules: "hudi-spark-datasource/hudi-spark3.2.x" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark3.3" | |
| sparkModules: "hudi-spark-datasource/hudi-spark3.3.x" | |
| - scalaProfile: "scala-2.12" | |
| sparkProfile: "spark3.4" | |
| sparkModules: "hudi-spark-datasource/hudi-spark3.4.x" | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Build Project | |
| env: | |
| SCALA_PROFILE: ${{ matrix.scalaProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| run: | |
| mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS | |
| - name: Quickstart Test | |
| env: | |
| SCALA_PROFILE: ${{ matrix.scalaProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| run: | |
| mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl hudi-examples/hudi-examples-spark $MVN_ARGS | |
| - name: UT - Common & Spark | |
| env: | |
| SCALA_PROFILE: ${{ matrix.scalaProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_MODULES: ${{ matrix.sparkModules }} | |
| if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI | |
| run: | |
| mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS | |
| - name: FT - Spark | |
| env: | |
| SCALA_PROFILE: ${{ matrix.scalaProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_MODULES: ${{ matrix.sparkModules }} | |
| if: ${{ !endsWith(env.SPARK_PROFILE, '3.2') }} # skip test spark 3.2 as it's covered by Azure CI | |
| run: | |
| mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS | |
| test-flink: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - flinkProfile: "flink1.13" | |
| - flinkProfile: "flink1.14" | |
| - flinkProfile: "flink1.15" | |
| - flinkProfile: "flink1.16" | |
| - flinkProfile: "flink1.17" | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Build Project | |
| env: | |
| SCALA_PROFILE: 'scala-2.12' | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| run: | |
| mvn clean install -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink -am -Davro.version=1.10.0 -DskipTests=true $MVN_ARGS | |
| - name: Quickstart Test | |
| env: | |
| SCALA_PROFILE: 'scala-2.12' | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| run: | |
| mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-examples/hudi-examples-flink $MVN_ARGS | |
| - name: Integration Test | |
| env: | |
| SCALA_PROFILE: 'scala-2.12' | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| if: ${{ endsWith(env.FLINK_PROFILE, '1.17') }} | |
| run: | | |
| mvn clean install -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version=1.10.0 -DskipTests=true $MVN_ARGS | |
| mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink $MVN_ARGS | |
| validate-bundles: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - flinkProfile: 'flink1.17' | |
| sparkProfile: 'spark3.4' | |
| sparkRuntime: 'spark3.4.0' | |
| - flinkProfile: 'flink1.17' | |
| sparkProfile: 'spark3.3' | |
| sparkRuntime: 'spark3.3.2' | |
| - flinkProfile: 'flink1.16' | |
| sparkProfile: 'spark3.3' | |
| sparkRuntime: 'spark3.3.2' | |
| - flinkProfile: 'flink1.15' | |
| sparkProfile: 'spark3.3' | |
| sparkRuntime: 'spark3.3.1' | |
| - flinkProfile: 'flink1.14' | |
| sparkProfile: 'spark3.2' | |
| sparkRuntime: 'spark3.2.3' | |
| - flinkProfile: 'flink1.13' | |
| sparkProfile: 'spark3.1' | |
| sparkRuntime: 'spark3.1.3' | |
| - flinkProfile: 'flink1.14' | |
| sparkProfile: 'spark3.0' | |
| sparkRuntime: 'spark3.0.2' | |
| - flinkProfile: 'flink1.13' | |
| sparkProfile: 'spark2.4' | |
| sparkRuntime: 'spark2.4.8' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Build Project | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| run: | | |
| mvn clean package -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS | |
| # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708. | |
| sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources | |
| mvn clean package -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 | |
| - name: IT - Bundle Validation - OpenJDK 8 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on Spark 3 | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk8 | |
| - name: IT - Bundle Validation - OpenJDK 11 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| if: ${{ env.SPARK_PROFILE >= 'spark3' }} # Only run validation on Spark 3 | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk11 | |
| - name: IT - Bundle Validation - OpenJDK 17 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| SCALA_PROFILE: 'scala-2.12' | |
| if: ${{ env.SPARK_PROFILE >= 'spark3.3' }} # Only Spark 3.3 and above support Java 17 | |
| run: | | |
| HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk17 | |
| validate-release-candidate-bundles: | |
| if: false | |
| runs-on: ubuntu-latest | |
| env: | |
| HUDI_VERSION: 0.13.1-rcx | |
| STAGING_REPO_NUM: 1123 | |
| strategy: | |
| matrix: | |
| include: | |
| - flinkProfile: 'flink1.16' | |
| sparkProfile: 'spark3' | |
| sparkRuntime: 'spark3.3.2' | |
| - flinkProfile: 'flink1.15' | |
| sparkProfile: 'spark3.3' | |
| sparkRuntime: 'spark3.3.1' | |
| - flinkProfile: 'flink1.14' | |
| sparkProfile: 'spark3.2' | |
| sparkRuntime: 'spark3.2.3' | |
| - flinkProfile: 'flink1.13' | |
| sparkProfile: 'spark3.1' | |
| sparkRuntime: 'spark3.1.3' | |
| - flinkProfile: 'flink1.13' | |
| sparkProfile: 'spark' | |
| sparkRuntime: 'spark2.4.8' | |
| - flinkProfile: 'flink1.13' | |
| sparkProfile: 'spark2.4' | |
| sparkRuntime: 'spark2.4.8' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: IT - Bundle Validation - OpenJDK 8 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| run: | | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk8 $STAGING_REPO_NUM | |
| - name: IT - Bundle Validation - OpenJDK 11 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| if: ${{ startsWith(env.SPARK_PROFILE, 'spark3') }} # Only Spark 3.x supports Java 11 as of now | |
| run: | | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk11 $STAGING_REPO_NUM | |
| - name: IT - Bundle Validation - OpenJDK 17 | |
| env: | |
| FLINK_PROFILE: ${{ matrix.flinkProfile }} | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_RUNTIME: ${{ matrix.sparkRuntime }} | |
| if: ${{ endsWith(env.SPARK_PROFILE, '3.3') }} # Only Spark 3.3 supports Java 17 as of now | |
| run: | | |
| ./packaging/bundle-validation/ci_run.sh $HUDI_VERSION openjdk17 $STAGING_REPO_NUM | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - sparkProfile: 'spark2.4' | |
| sparkArchive: 'spark-2.4.4/spark-2.4.4-bin-hadoop2.7.tgz' | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| architecture: x64 | |
| - name: Build Project | |
| env: | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SCALA_PROFILE: '-Dscala-2.11 -Dscala.binary.version=2.11' | |
| run: | |
| mvn clean install $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -DskipTests=true $MVN_ARGS | |
| - name: 'UT integ-test' | |
| env: | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SCALA_PROFILE: '-Dscala-2.11 -Dscala.binary.version=2.11' | |
| run: | |
| mvn test $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -DskipUTs=false -DskipITs=true -pl hudi-integ-test $MVN_ARGS | |
| - name: 'IT' | |
| env: | |
| SPARK_PROFILE: ${{ matrix.sparkProfile }} | |
| SPARK_ARCHIVE: ${{ matrix.sparkArchive }} | |
| SCALA_PROFILE: '-Dscala-2.11 -Dscala.binary.version=2.11' | |
| run: | | |
| echo "Downloading $SPARK_ARCHIVE" | |
| curl https://archive.apache.org/dist/spark/$SPARK_ARCHIVE --create-dirs -o $GITHUB_WORKSPACE/$SPARK_ARCHIVE | |
| tar -xvf $GITHUB_WORKSPACE/$SPARK_ARCHIVE -C $GITHUB_WORKSPACE/ | |
| mkdir /tmp/spark-events/ | |
| SPARK_ARCHIVE_BASENAME=$(basename $SPARK_ARCHIVE) | |
| export SPARK_HOME=$GITHUB_WORKSPACE/${SPARK_ARCHIVE_BASENAME%.*} | |
| mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl !hudi-flink-datasource/hudi-flink $MVN_ARGS |