Skip to content

Commit

Permalink
Build docker image with hops hadoop distribution
Browse files Browse the repository at this point in the history
  • Loading branch information
gibchikafa committed Jun 25, 2024
1 parent 4944d54 commit dbee648
Showing 6 changed files with 81 additions and 34 deletions.
43 changes: 43 additions & 0 deletions Jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
pipeline {
agent {
label "local"
}
environment {
VERSION = readFile "${env.WORKSPACE}/version.txt"
BUILD_BRANCH = readFile "${env.WORKSPACE}/build_branch.txt"
}
stages {
stage("build and publish") {
agent {
label "local"
}
steps {
withCredentials([usernamePassword(credentialsId: 'cred', passwordVariable: 'NEXUS_CREDS_PSW', usernameVariable: 'NEXUS_CREDS_USR')]) {
sh """
set -ex
echo "Building RSS version ${VERSION} on branch ${BUILD_BRANCH}"
echo "NEXUS_CREDS_USR: ${NEXUS_CREDS_USR}, NEXUS_CREDS_PSW: ${NEXUS_CREDS_PSW}"
docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} docker.hops.works
git checkout ${BUILD_BRANCH}
./build_distribution.sh --spark-profile spark3 --hadoop-profile hadoop3.2 --without-dashboard
cd deploy/kubernetes/docker || exit
./build.sh --hadoop-version 3.2.0.13-EE-SNAPSHOT --registry docker.hops.works --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW
cd ../../..
mkdir -p /opt/repository/master/rss/${VERSION}/
cp client/target/rss-client-${VERSION}.jar /opt/repository/master/rss/${VERSION}/
"""
}
}
}
}

post {
success {
build job:'Remote shuffle services', parameters: [
string(name: 'image', value: "rss"),
string(name: 'branch', value: "master")
]
}
}
}
1 change: 1 addition & 0 deletions build_branch.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
branch-0.9
27 changes: 7 additions & 20 deletions deploy/kubernetes/docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,19 +1,4 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# syntax=docker/dockerfile:experimental

ARG BASE_IMAGE=uniffle-base:latest
FROM ${BASE_IMAGE}
@@ -24,6 +9,7 @@ ARG RSS_VERSION
ARG AUTHOR=unknown
ARG GIT_BRANCH=unknown
ARG GIT_COMMIT=unknown
ARG HADOOP_FILE=hops-$HADOOP_VERSION.tgz

LABEL author=${AUTHOR}
LABEL git-branch=${GIT_BRANCH}
@@ -44,10 +30,11 @@ COPY rss-env.sh /data/rssadmin/rss/bin

COPY start.sh /data/rssadmin/rss/bin

COPY hadoop-${HADOOP_VERSION}.tar.gz /data/rssadmin
RUN tar -zxvf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz -C /data/rssadmin
RUN mv /data/rssadmin/hadoop-${HADOOP_VERSION} /data/rssadmin/hadoop
RUN rm -rf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz
COPY $HADOOP_FILE /data/rssadmin
RUN mkdir -p /data/rssadmin/hops-${HADOOP_VERSION}
RUN tar -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION}
RUN mv /data/rssadmin/hops-${HADOOP_VERSION} /data/rssadmin/hadoop
RUN rm -rf /data/rssadmin/$HADOOP_FILE
COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop

ENV RSS_VERSION ${RSS_VERSION}
2 changes: 1 addition & 1 deletion deploy/kubernetes/docker/base/debian/Dockerfile
Original file line number Diff line number Diff line change
@@ -20,5 +20,5 @@ ARG BASE_IMAGE=eclipse-temurin:11-jdk
FROM ${BASE_IMAGE}

RUN apt-get update && \
apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping && \
apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping curl && \
apt-get clean && rm -rf /var/lib/apt/lists/*
41 changes: 28 additions & 13 deletions deploy/kubernetes/docker/build.sh
Original file line number Diff line number Diff line change
@@ -29,14 +29,14 @@ function exit_with_usage() {
echo "+------------------------------------------------------------------------------------------------------+"
echo "| ./build.sh [--hadoop-version <hadoop version>] [--registry <registry url>] [--author <author name>] |"
echo "| [--base-os-distribution <os distribution>] [--base-image <base image url>] |"
echo "| [--push-image <true|false>] [--apache-mirror <apache mirror url>] |"
echo "| [--push-image <true|false>] [--apache-mirror <apache mirror url>] [--nexus-user <nexus username>]
[--nexus-password <nexus password>] |"
echo "+------------------------------------------------------------------------------------------------------+"
exit 1
}

REGISTRY="docker.io/library"
HADOOP_VERSION=2.8.5
HADOOP_SHORT_VERSION=$(echo $HADOOP_VERSION | awk -F "." '{print $1"."$2}')
HADOOP_VERSION=3.2.0.13-SNAPSHOT
AUTHOR=$(whoami)
# If you are based in China, you could pass --apache-mirror <a_mirror_url> when building this.
APACHE_MIRROR="https://dlcdn.apache.org"
@@ -70,6 +70,14 @@ while (( "$#" )); do
PUSH_IMAGE="$2"
shift
;;
--nexus-user)
NEXUS_USER="$2"
shift
;;
--nexus-password)
NEXUS_PASSWORD="$2"
shift
;;
--help)
exit_with_usage
;;
@@ -92,6 +100,8 @@ while (( "$#" )); do
shift
done

HADOOP_SHORT_VERSION=$(echo $HADOOP_VERSION | awk -F "." '{print $1"."$2}')

if [ -z "$BASE_IMAGE" ]; then
echo "start building base image: uniffle-base"
docker build -t "uniffle-base:latest" \
@@ -101,14 +111,15 @@ else
echo "using base image(${BASE_IMAGE}) to build rss server"
fi


HADOOP_FILE=hadoop-${HADOOP_VERSION}.tar.gz
ARCHIVE_HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
HADOOP_URL=${APACHE_MIRROR}/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
echo "HADOOP_URL is either ${HADOOP_URL} or ${ARCHIVE_HADOOP_URL}"
if [ ! -e "$HADOOP_FILE" ]; \
then wget "${HADOOP_URL}" || wget "$ARCHIVE_HADOOP_URL"; \
else echo "${HADOOP_FILE} has been downloaded"; \
HADOOP_FILE=hops-$HADOOP_VERSION.tgz
if [ ! -e "$HADOOP_FILE" ]; then
if [[ -n "$NEXUS_USER" ]] && [[ -n "$NEXUS_PASSWORD" ]] ; then
wget --user $NEXUS_USER --password $NEXUS_PASSWORD https://nexus.hops.works/repository/hopshadoop/$HADOOP_FILE
else
wget https://repo.hops.works/master/$HADOOP_FILE
fi
else
echo "${HADOOP_FILE} has been downloaded";
fi

RSS_DIR=../../..
@@ -126,17 +137,21 @@ cp "$RSS_DIR/$RSS_FILE" .

GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
GIT_COMMIT=$(git describe --dirty --always --tags | sed 's/-/./g')
echo "image version: ${IMAGE_VERSION:=$RSS_VERSION-$GIT_COMMIT}"
IMAGE_VERSION=$(cat $RSS_DIR/version.txt | awk '{$1=$1; print}' | sed '/^$/d')
IMAGE=$REGISTRY/rss-server:$IMAGE_VERSION
echo "image version: ${IMAGE_VERSION}"
IMAGE=$REGISTRY/rss:$IMAGE_VERSION
echo "building image: $IMAGE"
docker build --network=host -t "$IMAGE" \

DOCKER_BUILDKIT=1 docker build --network=host -t "$IMAGE" \
--build-arg RSS_VERSION="$RSS_VERSION" \
--build-arg HADOOP_VERSION="$HADOOP_VERSION" \
--build-arg HADOOP_SHORT_VERSION="$HADOOP_SHORT_VERSION" \
--build-arg AUTHOR="$AUTHOR" \
--build-arg GIT_COMMIT="$GIT_COMMIT" \
--build-arg GIT_BRANCH="$GIT_BRANCH" \
--build-arg BASE_IMAGE="$BASE_IMAGE" \
--build-arg HADOOP_FILE="$HADOOP_FILE" \
-f Dockerfile --no-cache .

if [ x"${PUSH_IMAGE}" == x"true" ]; then
1 change: 1 addition & 0 deletions version.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0.9.0

0 comments on commit dbee648

Please sign in to comment.