diff --git a/Jenkinsfile b/Jenkinsfile index 8dbf2d78c0..fb624b01c2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { DOCKER_REGISTRY = "docker.hops.works" CONTROLLER_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-controller:${VERSION}" WEBHOOK_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-webhook:${VERSION}" + UNIFFLE_VERSION = "0.10.0-SNAPSHOT" } stages { stage("checkout") { @@ -31,13 +32,15 @@ pipeline { echo "Building RSS version ${VERSION} on branch ${BUILD_BRANCH}" docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} $DOCKER_REGISTRY - ./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2 + docker run --rm -v .:/incubator-uniffle -w /incubator-uniffle openjdk:8-jdk /bin/bash build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2 + + #./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2 cd deploy/kubernetes/docker || exit ./build.sh --hadoop-version 3.2.0.15-EE-SNAPSHOT --registry $DOCKER_REGISTRY --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW cd ../../.. mkdir -p /opt/repository/master/rss/${VERSION}/ - cp client-spark/spark3-shaded/target/rss-client-spark3-shaded-${VERSION}.jar /opt/repository/master/rss/${VERSION}/ + cp client-spark/spark3-shaded/target/rss-client-spark3-shaded-${UNIFFLE_VERSION}.jar /opt/repository/master/rss/${VERSION}/rss-client-spark3-shaded-${VERSION}.jar # build the controller and webhook images cd deploy/kubernetes/operator || exit 1 diff --git a/deploy/kubernetes/docker/Dockerfile b/deploy/kubernetes/docker/Dockerfile index 1874e94846..25a3e8627c 100644 --- a/deploy/kubernetes/docker/Dockerfile +++ b/deploy/kubernetes/docker/Dockerfile @@ -21,13 +21,13 @@ RUN chown -R rssadmin:rssadmin /data RUN chmod -R 777 /data USER rssadmin -COPY rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz /data/rssadmin -RUN tar -xvf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz -C /data/rssadmin +COPY --chown=rssadmin:rssadmin rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz /data/rssadmin +RUN tar --no-same-owner -xvf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz -C /data/rssadmin RUN mv /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION} /data/rssadmin/rss RUN rm /data/rssadmin/rss/conf/rss-env.sh RUN rm -rf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz -COPY rss-env.sh /data/rssadmin/rss/conf +COPY --chown=rssadmin:rssadmin rss-env.sh /data/rssadmin/rss/conf RUN chmod -R 777 /data/rssadmin/rss @@ -35,12 +35,12 @@ COPY --chmod=755 rss-env.sh /data/rssadmin/rss/bin COPY --chmod=755 start.sh /data/rssadmin/rss/bin -COPY $HADOOP_FILE /data/rssadmin +COPY --chown=rssadmin:rssadmin $HADOOP_FILE /data/rssadmin RUN mkdir -p /data/rssadmin/hops-${HADOOP_VERSION} -RUN tar -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION} +RUN tar --no-same-owner -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION} RUN mv /data/rssadmin/hops-${HADOOP_VERSION} /data/rssadmin/hadoop RUN rm -rf /data/rssadmin/$HADOOP_FILE -COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop +COPY --chown=rssadmin:rssadmin hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop ENV RSS_VERSION ${RSS_VERSION} diff --git a/deploy/kubernetes/docker/start.sh b/deploy/kubernetes/docker/start.sh index c3602897ef..6cb0e268a7 100644 --- a/deploy/kubernetes/docker/start.sh +++ b/deploy/kubernetes/docker/start.sh @@ -30,10 +30,16 @@ if [ "$SERVICE_NAME" == "coordinator" ];then bash ${basedir}/bin/start-coordinator.sh & sleep 10 + max_retries=10 + retries=0 while : ; do pid=$(lsof -i:"${COORDINATOR_RPC_PORT}" -sTCP:LISTEN) if [ "$pid" = "" ]; then - break + retries=$((retries+1)) + if [ $retries -gt $max_retries ]; then + sleep 5 + break + fi else echo "coordinator pid:$pid is alive" sleep 10 @@ -45,10 +51,16 @@ if [ "$SERVICE_NAME" == "server" ];then bash ${basedir}/bin/start-shuffle-server.sh & sleep 10 + max_retries=10 + retries=0 while : ; do pid=$(lsof -i:"$SERVER_RPC_PORT" -sTCP:LISTEN) if [ "$pid" = "" ]; then - break + retries=$((retries+1)) + if [ $retries -gt $max_retries ]; then + sleep 5 + break + fi else echo "shuffle server pid:$pid is alive" sleep 10 @@ -58,13 +70,22 @@ fi if [ "$SERVICE_NAME" == "dashboard" ]; then bash ${basedir}/bin/start-dashboard.sh + sleep 10 + max_retries=10 + retries=0 while : ; do pid=$(lsof -i:"$DASHBOARD_PORT" -sTCP:LISTEN) if [ "$pid" = "" ]; then - break + retries=$((retries+1)) + if [ $retries -gt $max_retries ]; then + echo "dashboard pid:$pid is not alive" + sleep 5 + break + fi else echo "dashboard pid:$pid is alive" sleep 10 + retries=0 fi done fi