diff --git a/util/cron/publish-docker-images.bash b/util/cron/publish-docker-images.bash deleted file mode 100755 index e532c8d1217b..000000000000 --- a/util/cron/publish-docker-images.bash +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash - -# This script will build the docker images for 'amd' and 'arm' platforms using buildx and publish the images to the docker registry - -# Check if the script is run with correct arguement if not fail -required_vars=(docker_repository image_version username password) -for var in ${required_vars[@]}; do - if [ -z "${!var}" ] ; then - echo "${var} must be set." - exit 1 - fi -done - -echo "RepositoryName: $docker_repository" -echo "imageVersion: $image_version" - -CWD=$(cd $(dirname $0) ; pwd) -source $CWD/common.bash -export CHPL_HOME=$(cd $CWD/../.. ; pwd) -log_info "Setting CHPL_HOME to: ${CHPL_HOME}" - -# build_publish will build multi platform chapel docker images, tags them, and pushes the images to the docker repository . - -build_publish(){ - -local registry="$1" -local imageName="$2" -local version="$3" - -# the below buildx command will build images for amd and arm, tags with the tags specified, and pushes it to the docker repository($registry) -docker buildx build --platform=linux/amd64,linux/arm64 . --push -t $registry/$imageName:$version -t $registry/$imageName:latest - -if [ $? -ne 0 ] -then - echo "docker publish using buildx failed " - exit 1 -else - echo "docker publish using buildx succeeded " -fi - -} - -# Get the repository name and chapel version, Build chapel docker images and push to docker hub repository . -#build and publish chapel docker image -docker login -u $username -p $password -if [ $? -ne 0 ] -then - echo " Docker login failed " - exit 1 -else - echo "docker login succeeded " -fi - -cd $CHPL_HOME -build_publish $docker_repository chapel $image_version - -docker login -u $username -p $password -if [ $? -ne 0 ] -then - echo " Docker login failed " - exit 1 -else - echo "docker login succeeded " -fi -#build and publish chapel-gasnet docker image -cd $CHPL_HOME/util/packaging/docker/gasnet -build_publish $docker_repository chapel-gasnet $image_version - -docker login -u $username -p $password -if [ $? -ne 0 ] -then - echo " Docker login failed " - exit 1 -else - echo "docker login succeeded " -fi -#build and publish chapel-gasnet-smp docker image -cd $CHPL_HOME/util/packaging/docker/gasnet-smp -build_publish $docker_repository chapel-gasnet-smp $image_version diff --git a/util/cron/test-docker.bash b/util/cron/test-docker.bash index d45f267f196f..2f2746e99bff 100755 --- a/util/cron/test-docker.bash +++ b/util/cron/test-docker.bash @@ -1,35 +1,93 @@ #!/usr/bin/env bash # -# This script will build, run, and verify chapel, gasnet, and gasnet-smp docker images. +# This script will build, (sanity) test, and push the chapel, chapel-gasnet, and +# chapel-gasnet-smp Docker images. Uses the 'nightly' tag by default, or +# 'latest' and a release version tag if specified. +# +# Assumes Docker is already running on the system, logged into an account with +# appropriate permissions to push the images. +# +# Expected environment variables: +# - RELEASE_VERSION (optional): If set, will also push the image tagged as +# 'latest' and this version. Should match version in release branch name. + CWD=$(cd $(dirname $0) ; pwd) source $CWD/common.bash export CHPL_HOME=$(cd $CWD/../.. ; pwd) log_info "Setting CHPL_HOME to: ${CHPL_HOME}" +export CHPL_NIGHTLY_TEST_CONFIG_NAME="docker" -# build_image function takes image name and docker script location as arguments. -# Builds the image with the name from arg$1, runs the container and execute the install and verify script located in the location $2. -build_image() { - local imageName="$1" +# BEGIN FUNCTIONS + +# Patch the Dockerfile to build FROM the nightly image instead of latest. +# Assumes the Dockerfile is available at ./Dockerfile. +# Arguments are forwarded to `patch` command. +dockerfile_nightly_patch() { + local patch_args="$*" + + local nightlypatch=" +1c1 +< FROM chapel/chapel:latest +--- +> FROM chapel/chapel:nightly +" + + patch $patch_args ./Dockerfile << EOF +$nightlypatch +EOF +} + +# Build, test, and push a Docker image. +# Args: +# - image name without tag +# - test script location +# - release version tag to use (optional, builds nightly otherwise) +update_image() { + local baseImageName="$1" local script="$2" - # Remove any existing image with the tag before building docker image - docker image rm --force $imageName + local release_tag="$3" + + # Use specified release version tag, or 'nightly' if not specified + local imageName="${baseImageName}:${release_tag:-nightly}" + + log_info "Starting $imageName..." + + # Remove any existing image with the tag before building + if [ -n "$release_tag" ] + then + docker image rm --force "$imageName" + fi - docker buildx build --platform=linux/amd64,linux/arm64 . --push -t $imageName + # Build and push image + # Note: We push before testing due to a limitation of Docker + # (https://github.com/docker/buildx/issues/59) which prevents loading a + # multi-arch image without pushing. This means we may push a broken nightly + # image before erroring out; it's important that release pushes come after + # all nightly pushes so we can't push a broken release image. + # Anna, 2024-10-07 + docker buildx build --platform=linux/amd64,linux/arm64 --push . -t "$imageName" BUILD_RESULT=$? + # Also push as 'latest' tag if this is a release build + if [ -n "$release_tag" ] + then + # Use base image name (without tag) to use Docker's default tag 'latest' + docker buildx build --platform=linux/amd64,linux/arm64 --push . -t "$baseImageName" + fi + if [ $BUILD_RESULT -ne 0 ] then echo "docker build failed for $imageName image" exit 1 fi - containerid= docker image ls | grep $imageName | awk '{print$3}' - cd ${CHPL_HOME}/util/cron + # Run test script inside container echo 'writeln("Hello, world!");' > hello.chpl - - docker run --rm -i $imageName < $script - + docker run --rm -i "$imageName" < "$script" CONTAINER_RUN=$? + # Clean up scratch chpl file for testing + rm hello.chpl + if [ $CONTAINER_RUN -ne 0 ] then echo "docker commands failed inside chapel $imageName container" @@ -37,32 +95,53 @@ build_image() { else echo "docker commands succeeded inside chapel $imageName container" fi + + log_info "Completed $imageName" } -# Patch the Dockerfile to build FROM the nightly image instead of latest. -# Assumes the Dockerfile is available at ./Dockerfile. -dockerfile_nightly_patch() { - local nightlypatch=" -1c1 -< FROM chapel/chapel:latest ---- -> FROM chapel/chapel:nightly -" - patch ./Dockerfile << EOF -$nightlypatch -EOF +# Build, test, and push all Chapel Docker images. +# Args: +# - release version tag to use (optional, builds nightly otherwise) +update_all_images() { + local release_tag="$1" + + cd "$CHPL_HOME" + update_image chapel/chapel "${CHPL_HOME}/util/cron/docker-chapel.bash" "$release_tag" + + cd "$CHPL_HOME/util/packaging/docker/gasnet" + dockerfile_nightly_patch + update_image chapel/chapel-gasnet "${CHPL_HOME}/util/cron/docker-gasnet.bash" "$release_tag" + # Clean up after patch changes + dockerfile_nightly_patch -R + + cd "$CHPL_HOME/util/packaging/docker/gasnet-smp" + dockerfile_nightly_patch + update_image chapel/chapel-gasnet-smp "${CHPL_HOME}/util/cron/docker-gasnet.bash" "$release_tag" + # Clean up after patch changes + dockerfile_nightly_patch -R } +# END FUNCTIONS -# Build chapel Docker images -cd $CHPL_HOME -build_image chapel/chapel:nightly ${CHPL_HOME}/util/cron/docker-chapel.bash -cd $CHPL_HOME/util/packaging/docker/gasnet -dockerfile_nightly_patch -build_image chapel/chapel-gasnet:nightly ${CHPL_HOME}/util/cron/docker-gasnet.bash +if [ -n "$RELEASE_VERSION" ] +then + log_info "Building and pushing nightly and release-tagged images for version: $RELEASE_VERSION" + release_branch="release/$RELEASE_VERSION" + if [ "$(git rev-parse --abbrev-ref HEAD)" != "$release_branch" ] + then + log_error "Not on expected release branch $release_branch for version $RELEASE_VERSION, aborting" + exit 1 + fi +else + log_info "Building and pushing nightly images" +fi -cd $CHPL_HOME/util/packaging/docker/gasnet-smp -dockerfile_nightly_patch -build_image chapel/chapel-gasnet-smp:nightly ${CHPL_HOME}/util/cron/docker-gasnet.bash +# Build and push nightly images +update_all_images -export CHPL_NIGHTLY_TEST_CONFIG_NAME="docker" +# Build and push release-tagged images, if RELEASE_VERSION was specified. +# Runs after all nightly images, to abort if any fail. +if [ -n "$RELEASE_VERSION" ] +then + update_all_images "$RELEASE_VERSION" +fi