diff --git a/Jenkinsfile b/Jenkinsfile index 8dbf2d78c0..5b275b1e62 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { DOCKER_REGISTRY = "docker.hops.works" CONTROLLER_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-controller:${VERSION}" WEBHOOK_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-webhook:${VERSION}" + UNIFFLE_VERSION = "0.10.0-SNAPSHOT" } stages { stage("checkout") { @@ -31,13 +32,15 @@ pipeline { echo "Building RSS version ${VERSION} on branch ${BUILD_BRANCH}" docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} $DOCKER_REGISTRY - ./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2 + docker run --rm -v .:/incubator-uniffle -w /incubator-uniffle openjdk:8-jdk /bin/bash build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2 + + #./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-mr --without-tez --without-spark2 cd deploy/kubernetes/docker || exit ./build.sh --hadoop-version 3.2.0.15-EE-SNAPSHOT --registry $DOCKER_REGISTRY --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW cd ../../.. mkdir -p /opt/repository/master/rss/${VERSION}/ - cp client-spark/spark3-shaded/target/rss-client-spark3-shaded-${VERSION}.jar /opt/repository/master/rss/${VERSION}/ + cp client-spark/spark3-shaded/target/rss-client-spark3-shaded-${UNIFFLE_VERSION}.jar /opt/repository/master/rss/${VERSION}/ # build the controller and webhook images cd deploy/kubernetes/operator || exit 1 diff --git a/deploy/kubernetes/docker/Dockerfile b/deploy/kubernetes/docker/Dockerfile index 1874e94846..25a3e8627c 100644 --- a/deploy/kubernetes/docker/Dockerfile +++ b/deploy/kubernetes/docker/Dockerfile @@ -21,13 +21,13 @@ RUN chown -R rssadmin:rssadmin /data RUN chmod -R 777 /data USER rssadmin -COPY rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz /data/rssadmin -RUN tar -xvf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz -C /data/rssadmin +COPY --chown=rssadmin:rssadmin rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz /data/rssadmin +RUN tar --no-same-owner -xvf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz -C /data/rssadmin RUN mv /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION} /data/rssadmin/rss RUN rm /data/rssadmin/rss/conf/rss-env.sh RUN rm -rf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz -COPY rss-env.sh /data/rssadmin/rss/conf +COPY --chown=rssadmin:rssadmin rss-env.sh /data/rssadmin/rss/conf RUN chmod -R 777 /data/rssadmin/rss @@ -35,12 +35,12 @@ COPY --chmod=755 rss-env.sh /data/rssadmin/rss/bin COPY --chmod=755 start.sh /data/rssadmin/rss/bin -COPY $HADOOP_FILE /data/rssadmin +COPY --chown=rssadmin:rssadmin $HADOOP_FILE /data/rssadmin RUN mkdir -p /data/rssadmin/hops-${HADOOP_VERSION} -RUN tar -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION} +RUN tar --no-same-owner -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION} RUN mv /data/rssadmin/hops-${HADOOP_VERSION} /data/rssadmin/hadoop RUN rm -rf /data/rssadmin/$HADOOP_FILE -COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop +COPY --chown=rssadmin:rssadmin hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop ENV RSS_VERSION ${RSS_VERSION}