Skip to content

Commit

Permalink
[HUDI-5357] Fix release build commands (#7501)
Browse files Browse the repository at this point in the history
  • Loading branch information
@xushiyan @satishkotha
xushiyan authored and satishkotha committed Dec 21, 2022
1 parent 94db72e commit 975eb91
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 17 deletions.
37 changes: 23 additions & 14 deletions scripts/release/deploy_staging_jars.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ fi

BUNDLE_MODULES=$(find -s packaging -name 'hudi-*-bundle' -type d)
BUNDLE_MODULES_EXCLUDED="-${BUNDLE_MODULES//$'\n'/,-}"
BUNDLE_MODULES_EXCLUDED="-packaging/hudi-aws-bundle,-packaging/hudi-datahub-sync-bundle,-packaging/hudi-flink-bundle,-packaging/hudi-gcp-bundle,-packaging/hudi-integ-test-bundle,-packaging/hudi-kafka-connect-bundle"

declare -a ALL_VERSION_OPTS=(
# upload all module jars and bundle jars
Expand All @@ -48,25 +49,25 @@ declare -a ALL_VERSION_OPTS=(
"-Dscala-2.12 -Dspark3.1" # this profile goes last in this section to ensure bundles use avro 1.8

# spark bundles
"-Dscala-2.11 -Dspark2.4 -pl packaging/hudi-spark-bundle"
"-Dscala-2.12 -Dspark2.4 -pl packaging/hudi-spark-bundle"
"-Dscala-2.12 -Dspark3.3 -pl packaging/hudi-spark-bundle"
"-Dscala-2.12 -Dspark3.2 -pl packaging/hudi-spark-bundle"
"-Dscala-2.12 -Dspark3.1 -pl packaging/hudi-spark-bundle"
"-Dscala-2.11 -Dspark2.4 -pl packaging/hudi-spark-bundle -am"
"-Dscala-2.12 -Dspark2.4 -pl packaging/hudi-spark-bundle -am"
"-Dscala-2.12 -Dspark3.3 -pl packaging/hudi-spark-bundle -am"
"-Dscala-2.12 -Dspark3.2 -pl packaging/hudi-spark-bundle -am"
"-Dscala-2.12 -Dspark3.1 -pl packaging/hudi-spark-bundle -am"

# spark bundles (legacy) (not overwriting previous uploads as these jar names are unique)
"-Dscala-2.11 -Dspark2 -pl packaging/hudi-spark-bundle" # for legacy bundle name hudi-spark-bundle_2.11
"-Dscala-2.12 -Dspark2 -pl packaging/hudi-spark-bundle" # for legacy bundle name hudi-spark-bundle_2.12
"-Dscala-2.12 -Dspark3 -pl packaging/hudi-spark-bundle" # for legacy bundle name hudi-spark3-bundle_2.12
"-Dscala-2.11 -Dspark2 -pl packaging/hudi-spark-bundle -am" # for legacy bundle name hudi-spark-bundle_2.11
"-Dscala-2.12 -Dspark2 -pl packaging/hudi-spark-bundle -am" # for legacy bundle name hudi-spark-bundle_2.12
"-Dscala-2.12 -Dspark3 -pl packaging/hudi-spark-bundle -am" # for legacy bundle name hudi-spark3-bundle_2.12

# utilities bundles (legacy) (overwriting previous uploads)
"-Dscala-2.11 -Dspark2.4 -pl packaging/hudi-utilities-bundle" # utilities-bundle_2.11 is for spark 2.4 only
"-Dscala-2.12 -Dspark3.1 -pl packaging/hudi-utilities-bundle" # utilities-bundle_2.12 is for spark 3.1 only
"-Dscala-2.11 -Dspark2.4 -pl packaging/hudi-utilities-bundle -am" # utilities-bundle_2.11 is for spark 2.4 only
"-Dscala-2.12 -Dspark3.1 -pl packaging/hudi-utilities-bundle -am" # utilities-bundle_2.12 is for spark 3.1 only

# flink bundles (overwriting previous uploads)
"-Dscala-2.12 -Dflink1.13 -Davro.version=1.10.0 -pl packaging/hudi-flink-bundle"
"-Dscala-2.12 -Dflink1.14 -Davro.version=1.10.0 -pl packaging/hudi-flink-bundle"
"-Dscala-2.12 -Dflink1.15 -Davro.version=1.10.0 -pl packaging/hudi-flink-bundle"
"-Dscala-2.12 -Dflink1.13 -Davro.version=1.10.0 -pl packaging/hudi-flink-bundle -am"
"-Dscala-2.12 -Dflink1.14 -Davro.version=1.10.0 -pl packaging/hudi-flink-bundle -am"
"-Dscala-2.12 -Dflink1.15 -Davro.version=1.10.0 -pl packaging/hudi-flink-bundle -am"
)
printf -v joined "'%s'\n" "${ALL_VERSION_OPTS[@]}"

Expand Down Expand Up @@ -98,7 +99,15 @@ fi

for v in "${ALL_VERSION_OPTS[@]}"
do
echo "Deploying to repository.apache.org with version option ${v}"
# clean everything before any round of depoyment
#$MVN clean
#echo "Building with options ${v}"
#echo "install Command: $MVN install "$COMMON_OPTIONS" "${v}""
#$MVN install "${v}" "$COMMON_OPTIONS"
#echo "Deploying to repository.apache.org with version options $COMMON_OPTIONS ${v%-am}"
#echo "Command execute: $MVN clean install deploy "${v%-am}" "$COMMON_OPTIONS""
# remove `-am` option to only deploy intended modules
#$MVN clean deploy "$COMMON_OPTIONS" "${v%-am}"
COMMON_OPTIONS="${v} -DdeployArtifacts=true -DskipTests -DretryFailedDeploymentCount=10"
$MVN clean deploy $COMMON_OPTIONS
done
4 changes: 1 addition & 3 deletions scripts/release/validate_staged_bundles.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,10 @@ VERSION=$2
STAGING_REPO="https://repository.apache.org/content/repositories/${REPO}/org/apache/hudi"

declare -a BUNDLE_URLS=(

"${STAGING_REPO}/hudi-datahub-sync-bundle/${VERSION}/hudi-datahub-sync-bundle-${VERSION}.jar"
"${STAGING_REPO}/hudi-flink1.13-bundle/${VERSION}/hudi-flink1.13-bundle-${VERSION}.jar"
"${STAGING_REPO}/hudi-flink1.13-bundle/${VERSION}/hudi-flink1.13-bundle-${VERSION}.jar"
"${STAGING_REPO}/hudi-flink1.14-bundle/${VERSION}/hudi-flink1.14-bundle-${VERSION}.jar"
"${STAGING_REPO}/hudi-flink1.14-bundle/${VERSION}/hudi-flink1.14-bundle-${VERSION}.jar"
"${STAGING_REPO}/hudi-flink1.15-bundle/${VERSION}/hudi-flink1.15-bundle-${VERSION}.jar"
"${STAGING_REPO}/hudi-flink1.15-bundle/${VERSION}/hudi-flink1.15-bundle-${VERSION}.jar"
"${STAGING_REPO}/hudi-gcp-bundle/${VERSION}/hudi-gcp-bundle-${VERSION}.jar"
"${STAGING_REPO}/hudi-hadoop-mr-bundle/${VERSION}/hudi-hadoop-mr-bundle-${VERSION}.jar"
Expand Down

0 comments on commit 975eb91

Please sign in to comment.