diff --git a/util/cron/common-tarball.bash b/util/cron/common-tarball.bash index 787c2175f096..449bef3e95a1 100644 --- a/util/cron/common-tarball.bash +++ b/util/cron/common-tarball.bash @@ -5,6 +5,7 @@ source $CWD/common.bash export CHPL_HOME=$(cd $CWD/../.. ; pwd) log_info "Setting CHPL_HOME to: ${CHPL_HOME}" +# Load functions for getting version information from the source tree source ${CHPL_HOME}/util/build_configs/functions.bash major=$(get_src_major_version ${CHPL_HOME}) diff --git a/util/cron/docker.bash b/util/cron/docker.bash deleted file mode 100644 index e56e6b9ddf1b..000000000000 --- a/util/cron/docker.bash +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Common functions used for docker - -check_docker_install(){ - # check if docker desktop is installed in mac if not install it before running the tests. - if [[ -n "$(docker info --format '{{.Architecture}}' | grep 'aarch64')" ]]; then - echo "aarch64 Architecture found. Skipping installation ..." - else - echo "ERROR! Docker Desktop not installed:" - echo " * Install docker desktop from " - - fi - -} - -start_docker(){ - check_docker_install - # Start docker desktop and wait for it to respond - printf "Starting Docker for Mac"; - colima start; - - while (! docker stats --no-stream ); do - # Docker takes a few seconds to initialize - echo "Waiting for Docker to launch..." - sleep 60 - done -} - - diff --git a/util/cron/publish-docker-images.bash b/util/cron/publish-docker-images.bash index b2417d1b89d4..e532c8d1217b 100755 --- a/util/cron/publish-docker-images.bash +++ b/util/cron/publish-docker-images.bash @@ -16,19 +16,15 @@ echo "imageVersion: $image_version" CWD=$(cd $(dirname $0) ; pwd) source $CWD/common.bash -source $CWD/docker.bash export CHPL_HOME=$(cd $CWD/../.. ; pwd) log_info "Setting CHPL_HOME to: ${CHPL_HOME}" -start_docker - - # build_publish will build multi platform chapel docker images, tags them, and pushes the images to the docker repository . build_publish(){ local registry="$1" -local imageName="$2" +local imageName="$2" local version="$3" # the below buildx command will build images for amd and arm, tags with the tags specified, and pushes it to the docker repository($registry) @@ -36,48 +32,48 @@ docker buildx build --platform=linux/amd64,linux/arm64 . --push -t $registry/$im if [ $? -ne 0 ] then - echo "docker publish using buildx failed " + echo "docker publish using buildx failed " exit 1 else echo "docker publish using buildx succeeded " -fi +fi } -# Get the repository name and chapel version, Build chapel docker images and push to docker hub repository . +# Get the repository name and chapel version, Build chapel docker images and push to docker hub repository . #build and publish chapel docker image docker login -u $username -p $password -if [ $? -ne 0 ] +if [ $? -ne 0 ] then echo " Docker login failed " exit 1 else echo "docker login succeeded " -fi +fi cd $CHPL_HOME build_publish $docker_repository chapel $image_version docker login -u $username -p $password -if [ $? -ne 0 ] +if [ $? -ne 0 ] then echo " Docker login failed " exit 1 else echo "docker login succeeded " -fi +fi #build and publish chapel-gasnet docker image cd $CHPL_HOME/util/packaging/docker/gasnet build_publish $docker_repository chapel-gasnet $image_version docker login -u $username -p $password -if [ $? -ne 0 ] +if [ $? -ne 0 ] then echo " Docker login failed " exit 1 else echo "docker login succeeded " -fi +fi #build and publish chapel-gasnet-smp docker image cd $CHPL_HOME/util/packaging/docker/gasnet-smp build_publish $docker_repository chapel-gasnet-smp $image_version diff --git a/util/cron/test-docker.bash b/util/cron/test-docker.bash index 8d99a2ec6317..6abfe1f568d9 100755 --- a/util/cron/test-docker.bash +++ b/util/cron/test-docker.bash @@ -4,12 +4,9 @@ CWD=$(cd $(dirname $0) ; pwd) source $CWD/common.bash -source $CWD/docker.bash export CHPL_HOME=$(cd $CWD/../.. ; pwd) log_info "Setting CHPL_HOME to: ${CHPL_HOME}" -start_docker - # build_image function takes image name and docker script location as arguments. # Builds the image with the name from arg$1, runs the container and execute the install and verify script located in the location $2. build_image() { diff --git a/util/cron/test-homebrew-linux.bash b/util/cron/test-homebrew-linux.bash index 088bcd7434d9..4754e01b599d 100755 --- a/util/cron/test-homebrew-linux.bash +++ b/util/cron/test-homebrew-linux.bash @@ -30,39 +30,42 @@ cd $CHPL_HOME # replace the url and sha in the chapel formula with the url pointing to the tarball created and sha of the tarball. # run home-brew scripts to install chapel. -# gen_release $short_version +gen_release $short_version cp ${CHPL_HOME}/util/packaging/homebrew/chapel-main.rb ${CHPL_HOME}/util/packaging/homebrew/chapel.rb cd ${CHPL_HOME}/util/packaging/homebrew # Get the tarball from the root tar/ directory and replace the url in chapel.rb with the tarball location location="${CHPL_HOME}/tar/chapel-${short_version}.tar.gz" +# Bail early if the tarball doesn't exist +if [[ ! -f $location ]]; then + log_error "FATAL: Did not find tarball at $location" + exit 1 +fi log_info $location # Replace the url and sha256 in chapel.rb with the location of the tarball and sha256 of the tarball generated. -# create sed -i '' -e for macOS -sed_command="sed -i '' -e" -$sed_command "s#url.*#url \"file\:///$location\"#" chapel.rb + +# Get the sha256 of the tarball sha=($(shasum -a 256 $location)) sha256=${sha[0]} log_info $sha256 -$sed_command "1s/sha256.*/sha256 \"$sha256\"/;t" -e "1,/sha256.*/s//sha256 \"$sha256\"/" chapel.rb -#To mimic home-brew CI. Run home-brew chpl install inside a container. -# check if docker desktop is installed, if not fail the test. -# TODO: maybe change this for linux? -source ${CHPL_HOME}/util/cron/docker.bash -start_docker -# This mimics homebrew-ci -# This will test homebrew installation inside ubuntu VM using the lastest chapel.rb using the tarball built -cd ${CHPL_HOME}/util/packaging/homebrew +# create sed -i command +sed_command="sed -i.bak -e " +# # Replace the tarball location in the container where the tarball is copied over $sed_command "s#url.*#url \"file\:////home/linuxbrew/chapel-${short_version}.tar.gz\"#" chapel.rb +$sed_command "1s/sha256.*/sha256 \"$sha256\"/;t" -e "1,/sha256.*/s//sha256 \"$sha256\"/" chapel.rb + +# To mimic home-brew CI. Run homebrew chpl install inside a container. +# This will test homebrew installation inside ubuntu VM using the lastest chapel.rb using the tarball built +cd ${CHPL_HOME}/util/packaging/homebrew cp ${CHPL_HOME}/util/packaging/homebrew/chapel.rb ${CHPL_HOME}/util/packaging/docker/test cp $location ${CHPL_HOME}/util/packaging/docker/test -#This will start a docker container that is similar to the one used by homebrew-ci and test the homebrew installation inside it. +# This will start a docker container that is similar to the one used by homebrew-ci and test the homebrew installation inside it. source ${CHPL_HOME}/util/packaging/docker/test/homebrew_ci.bash export CHPL_NIGHTLY_TEST_CONFIG_NAME="homebrew-linux" diff --git a/util/cron/test-homebrew.bash b/util/cron/test-homebrew.bash index 805828236bd1..d9c793cb4011 100755 --- a/util/cron/test-homebrew.bash +++ b/util/cron/test-homebrew.bash @@ -6,8 +6,6 @@ # replace the url and sha in the chapel formula with the url pointing to the tarball created and sha of the tarball. # run home-brew scripts to install chapel. -# Create a tarball from current repo. -# The tarball is left in root of repo in tar/ directory. CWD=$(cd $(dirname $0) ; pwd) # common-tarball sets CHPL_HOME @@ -18,6 +16,8 @@ source $CWD/common-tarball.bash export CHPL_GEN_RELEASE_NO_CLONE=true export CHPL_LLVM=none +# $CWD/common.bash sets this to none, but Homebrew builds with native +export CHPL_TARGET_CPU=native log_info "Moving to ${CHPL_HOME}" cd $CHPL_HOME @@ -26,6 +26,8 @@ cd $CHPL_HOME # replace the url and sha in the chapel formula with the url pointing to the tarball created and sha of the tarball. # run home-brew scripts to install chapel. +# Create a tarball from current repo. +# The tarball is left in root of repo in tar/ directory. gen_release $short_version cp ${CHPL_HOME}/util/packaging/homebrew/chapel-main.rb ${CHPL_HOME}/util/packaging/homebrew/chapel.rb @@ -33,16 +35,23 @@ cd ${CHPL_HOME}/util/packaging/homebrew # Get the tarball from the root tar/ directory and replace the url in chapel.rb with the tarball location location="${CHPL_HOME}/tar/chapel-${short_version}.tar.gz" +# Bail early if the tarball doesn't exist +if [[ ! -f $location ]]; then + log_error "FATAL: Did not find tarball at ${location}" + exit 1 +fi + log_info $location # Replace the url and sha256 in chapel.rb with the location of the tarball and sha256 of the tarball generated. -# create sed -i '' -e for macOS -sed_command="sed -i '' -e" -$sed_command "s#url.*#url \"file\:///$location\"#" chapel.rb sha=($(shasum -a 256 $location)) sha256=${sha[0]} log_info $sha256 -$sed_command "1s/sha256.*/sha256 \"$sha256\"/;t" -e "1,/sha256.*/s//sha256 \"$sha256\"/" chapel.rb + +# create sed command +sed_command="sed -i.bak -e " +$sed_command "s#url.*#url \"file\:///$location\"#" chapel.rb +$sed_command "1s/sha256.*/sha256 \"$sha256\"/;t" -e "1,/sha256.*/s//sha256 \"$sha256\"/" chapel.rb # Test if homebrew install using the chapel formula works. brew upgrade diff --git a/util/packaging/docker/test/Dockerfile b/util/packaging/docker/test/Dockerfile index d693b6dcfdc1..616fc443eb11 100644 --- a/util/packaging/docker/test/Dockerfile +++ b/util/packaging/docker/test/Dockerfile @@ -1,7 +1,7 @@ # This is a container used to run homebrew-ci -# Get the homebrew ubuntu container, set the platform to avoid warnings when on M1 -FROM --platform=linux/amd64 ghcr.io/homebrew/ubuntu22.04:master +# Get the homebrew ubuntu container +FROM ghcr.io/homebrew/ubuntu22.04:master # Redundant to brew_install.bash but imporant for anyone working in this container # which can be ran without brew_install.bash diff --git a/util/packaging/docker/test/homebrew_ci.bash b/util/packaging/docker/test/homebrew_ci.bash index 0906ac7103be..c0af0d903b77 100755 --- a/util/packaging/docker/test/homebrew_ci.bash +++ b/util/packaging/docker/test/homebrew_ci.bash @@ -1,22 +1,21 @@ #!/usr/bin/env bash -#This script will build a docker image using the Dockerfile and will execute home-brew install scripts inside the container -CWD=$(cd $(dirname $0) ; pwd) -export CHPL_HOME=$(cd $CWD/../.. ; pwd) +#This script will build a docker image using the Dockerfile and will execute homebrew install scripts inside the container +cd ${CHPL_HOME}/util/packaging/docker/test -cd ${CHPL_HOME}/packaging/docker/test +# specify the platform as linux/amd64 to avoid warnings from aarch64 platforms # Remove image with name homebrew_ci before creating a fresh image to avoid failures. docker image rm --force homebrew_ci -docker build . --load -t homebrew_ci +docker build . --load --platform linux/amd64 -t homebrew_ci containerid= docker image ls | grep 'homebrew_ci' | awk '{print$3}' # Start the container and run a script to check homebrew install inside the running container -docker run --rm -i homebrew_ci /bin/bash < ${CHPL_HOME}/packaging/docker/test/brew_install.bash +docker run --platform linux/amd64 --rm -i homebrew_ci /bin/bash < ${CHPL_HOME}/util/packaging/docker/test/brew_install.bash CONTAINER_RUN=$? - if [ $CONTAINER_RUN -ne 0 ] - then - echo "brew test bot commands failed inside ubuntu container" - exit 1 - else - echo "brew test bot commands succeeded inside ubuntu container" - fi +if [ $CONTAINER_RUN -ne 0 ] +then + echo "brew test bot commands failed inside ubuntu container" + exit 1 + else + echo "brew test bot commands succeeded inside ubuntu container" +fi