Build uniffle in a openjdk:8-jdk image

logicalclocks · Dec 16, 2024 · 4f18bcb · 4f18bcb
1 parent 05e0dc7
commit 4f18bcb
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 11 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -8,6 +8,7 @@ pipeline {
         DOCKER_REGISTRY = "docker.hops.works"
         CONTROLLER_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-controller:${VERSION}"
         WEBHOOK_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-webhook:${VERSION}"
+        UNIFFLE_VERSION = "0.10.0-SNAPSHOT"
     }
     stages {
         stage("checkout") {
@@ -31,13 +32,15 @@ pipeline {
                     echo "Building RSS version ${VERSION} on branch ${BUILD_BRANCH}"
                     docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} $DOCKER_REGISTRY
 
-                    ./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2
+                    docker run --rm -v .:/incubator-uniffle -w /incubator-uniffle  openjdk:8-jdk /bin/bash build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2
+
+                    #./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2
                     cd deploy/kubernetes/docker ||  exit
                     ./build.sh --hadoop-version 3.2.0.15-EE-SNAPSHOT --registry $DOCKER_REGISTRY --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW
                     cd ../../..
 
                     mkdir -p /opt/repository/master/rss/${VERSION}/
-                    cp  client-spark/spark3-shaded/target/rss-client-spark3-shaded-${VERSION}.jar /opt/repository/master/rss/${VERSION}/
+                    cp  client-spark/spark3-shaded/target/rss-client-spark3-shaded-${UNIFFLE_VERSION}.jar /opt/repository/master/rss/${VERSION}/rss-client-spark3-shaded-${VERSION}.jar
 
                     # build the controller and webhook images
                     cd deploy/kubernetes/operator ||  exit 1

diff --git a/deploy/kubernetes/docker/Dockerfile b/deploy/kubernetes/docker/Dockerfile
@@ -21,26 +21,26 @@ RUN chown -R rssadmin:rssadmin /data
 RUN chmod -R 777 /data
 USER rssadmin
 
-COPY rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz /data/rssadmin
-RUN tar -xvf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz -C /data/rssadmin
+COPY --chown=rssadmin:rssadmin rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz /data/rssadmin
+RUN tar --no-same-owner -xvf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz -C /data/rssadmin
 RUN mv /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION} /data/rssadmin/rss
 RUN rm /data/rssadmin/rss/conf/rss-env.sh
 RUN rm -rf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz
 
-COPY rss-env.sh /data/rssadmin/rss/conf
+COPY --chown=rssadmin:rssadmin rss-env.sh /data/rssadmin/rss/conf
 
 RUN chmod -R 777 /data/rssadmin/rss
 
 COPY --chmod=755 rss-env.sh /data/rssadmin/rss/bin
 
 COPY --chmod=755 start.sh /data/rssadmin/rss/bin
 
-COPY $HADOOP_FILE /data/rssadmin
+COPY --chown=rssadmin:rssadmin $HADOOP_FILE /data/rssadmin
 RUN mkdir -p /data/rssadmin/hops-${HADOOP_VERSION}
-RUN tar -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION}
+RUN tar --no-same-owner -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION}
 RUN mv /data/rssadmin/hops-${HADOOP_VERSION} /data/rssadmin/hadoop
 RUN rm -rf /data/rssadmin/$HADOOP_FILE
-COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop
+COPY --chown=rssadmin:rssadmin hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop
 
 ENV RSS_VERSION ${RSS_VERSION}
 

diff --git a/deploy/kubernetes/docker/start.sh b/deploy/kubernetes/docker/start.sh
@@ -30,10 +30,16 @@ if [ "$SERVICE_NAME" == "coordinator" ];then
 
     bash ${basedir}/bin/start-coordinator.sh &
     sleep 10
+    max_retries=10
+    retries=0
     while : ; do
         pid=$(lsof -i:"${COORDINATOR_RPC_PORT}" -sTCP:LISTEN)
         if [ "$pid" = "" ]; then
-          break
+          retries=$((retries+1))
+          if [ $retries -gt $max_retries ]; then
+            sleep 5
+            break
+          fi
         else
           echo "coordinator pid:$pid is alive"
           sleep 10
@@ -45,10 +51,16 @@ if [ "$SERVICE_NAME" == "server" ];then
 
     bash ${basedir}/bin/start-shuffle-server.sh &
     sleep 10
+    max_retries=10
+    retries=0
     while : ; do
         pid=$(lsof -i:"$SERVER_RPC_PORT" -sTCP:LISTEN)
         if [ "$pid" = "" ]; then
-          break
+          retries=$((retries+1))
+          if [ $retries -gt $max_retries ]; then
+            sleep 5
+            break
+          fi
         else
           echo "shuffle server pid:$pid is alive"
           sleep 10
@@ -58,13 +70,22 @@ fi
 
 if [ "$SERVICE_NAME" == "dashboard" ]; then
     bash ${basedir}/bin/start-dashboard.sh
+    sleep 10
+    max_retries=10
+    retries=0
     while : ; do
         pid=$(lsof -i:"$DASHBOARD_PORT" -sTCP:LISTEN)
         if [ "$pid" = "" ]; then
-          break
+          retries=$((retries+1))
+          if [ $retries -gt $max_retries ]; then
+            echo "dashboard pid:$pid is not alive"
+            sleep 5
+            break
+          fi
         else
           echo "dashboard pid:$pid is alive"
           sleep 10
+          retries=0
         fi
     done
 fi