From 054db6e756e8d4522490fc8541a292fef27f8f1e Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 9 Jun 2017 22:44:28 -0700 Subject: [PATCH 01/71] Adding docker and docker-compose cli --- base/Dockerfile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/base/Dockerfile b/base/Dockerfile index 8b77872..5613147 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -9,6 +9,29 @@ RUN apk add --update --no-cache \ sudo \ && rm -rf /var/cache/apk/* +ARG GLIBC=2.25-r0 +RUN \ + # Install glibc libraries needed for docker-compose + apk update && apk add --no-cache openssl ca-certificates && \ + curl -L https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub && \ + curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ + apk add --no-cache glibc-$GLIBC.apk && rm glibc-$GLIBC.apk && \ + ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ + ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib + +ARG DOCKER_VERSION=17.05.0-ce +ARG DOCKER_COMPOSE_VERSION=1.13.0 +RUN \ + # Install docker cli + curl -L -O https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz && \ + tar -xzf docker-${DOCKER_VERSION}.tgz && \ + mv docker/docker /usr/local/bin && \ + rm -rf docker docker-${DOCKER_VERSION}.tgz && \ + # Install docker-compose cli + curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose && \ + # Make binaries executable + chmod -R +x /usr/local/bin + ARG AGENT_USER=agent ARG AGENT_HOME=/home/agent RUN \ From 5cd59b48e646d4888635b17a3666018998fc2801 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 21 Jun 2017 17:51:15 -0700 Subject: [PATCH 02/71] Remote docker.sock access over SSH tunnel --- base/bin/build-env | 27 ++++++++++++++++++++------- base/config/.ssh/config | 3 ++- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 7fbf023..2cefa33 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -20,6 +20,11 @@ export COMPOSE_PROJECT_NAME="$BITBUCKET_REPO_SLUG_SAFE-$BRANCH_NAME_SAFE" # Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with xip.io DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.xip.io}" +sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config + +# User ubuntu as a user by default +DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" +sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config # NOTE: The length of any one label in the domain name is limited to between 1 and 63 octets. export DOMAIN="$BRANCH_NAME_SAFE.$BITBUCKET_REPO_SLUG_SAFE.$DOCKSAL_HOST" @@ -28,13 +33,21 @@ echo "Configuring ssh access..." (umask 077 ; echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa) (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa) -# Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) -if [[ $DOCKSAL_HOST != "" ]]; then - sed -i "s/DOCKSAL_HOST/$DOCKSAL_HOST/g" $HOME/.ssh/config -else - sed -i "s/DOCKSAL_HOST/$DOCKSAL_HOST_IP/g" $HOME/.ssh/config -fi - # Configure git git config --global user.email "${CI_GIT_USER_EMAIL:-bitbucket-pipelines-agent@docksal.io}" git config --global user.name "${CI_GIT_USER_NAME:-Bitbucket Pipelines Agent}" + +# Initialize a tunnel to the Docker Engine on DOCKSAL_HOST +if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then + echo "Setting up a secure tunnel to the Docker Engine on $DOCKSAL_HOST..." + # Black magic! Remote docker.sock access over SSH tunnel + # Credits: + # https://docs.docker.com/docker-for-aws/deploy/#connecting-via-ssh + # https://gist.github.com/scy/6781836#gistcomment-1559506 + ssh -fM -NL localhost:2374:/var/run/docker.sock docker-host + + echo "Querying Docker Engine..." + docker --host localhost:2374 version + # Export local tunnel connection settings if it works + [[ $? == 0 ]] && export DOCKER_HOST="localhost:2374" +fi diff --git a/base/config/.ssh/config b/base/config/.ssh/config index 4ce81c2..932b797 100644 --- a/base/config/.ssh/config +++ b/base/config/.ssh/config @@ -6,8 +6,9 @@ Host * Host docker-host HostName DOCKSAL_HOST - User ubuntu + User DOCKSAL_HOST_USER StrictHostKeyChecking no UserKnownHostsFile=/dev/null LogLevel ERROR IdentityFile ~/.ssh/docksal_host_id_rsa + ControlPath ~/.ssh/docksal_host.ctl From 86f42710f276887abae98fc24908f39cb945acf2 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 27 Jun 2017 16:37:12 -0700 Subject: [PATCH 03/71] Optimize COPY in Dockerfile --- base/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index 5613147..bdd2879 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -47,11 +47,9 @@ RUN \ mkdir -p $AGENT_HOME/.ssh && \ mkdir -p $AGENT_HOME/build +COPY bin /usr/local/bin COPY config/.ssh/config $AGENT_HOME/.ssh/config -COPY bin/build-env /usr/local/bin/build-env -COPY bin/build-init /usr/local/bin/build-init -COPY bin/slack /usr/local/bin/slack # Fix permissions after COPY -RUN sudo chown -R $AGENT_USER:$AGENT_USER $AGENT_HOME +RUN sudo chown -R $AGENT_USER:$AGENT_USER $AGENT_HOME/.ssh/config WORKDIR $AGENT_HOME/build From f69e4c194c9c7d73ad8b506532c57bd3a34ed033 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 27 Jun 2017 16:37:39 -0700 Subject: [PATCH 04/71] Support for CircleCI 2.0 --- base/bin/build-env | 46 +++++++++++++++++++++++++++++++++++---------- base/bin/build-init | 2 +- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 2cefa33..365112c 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -4,38 +4,64 @@ BRANCH_NAME_LENGTH_LIMIT=35 +# Support for Bitbucket Pipelines +if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then + echo "Detected Bitbucket Pipelines build environment" + REPO_OWNER="$BITBUCKET_REPO_OWNER" + REPO_NAME="$BITBUCKET_REPO_SLUG" + BRANCH_NAME="$BITBUCKET_BRANCH" + COMMIT_HASH="$BITBUCKET_COMMIT" +fi + +# Support for CircleCI 2.0 +if [[ "$CIRCLECI" != "" ]]; then + echo "Detected CircleCI build environment" + REPO_OWNER="$CIRCLE_PROJECT_USERNAME" + REPO_NAME="$CIRCLE_PROJECT_REPONAME" + BRANCH_NAME="$CIRCLE_BRANCH" + COMMIT_HASH="$CIRCLE_SHA1" +fi + +# For debug purposes these variables can be set manually. +# If they are empty here, then we cannot proceed. +if [[ "$REPO_OWNER" == "" ]] || [[ "$REPO_NAME" == "" ]] || [[ "$BRANCH_NAME" == "" ]]; then + echo "Unsupported build environment! Quiting..." + exit 1 +fi + echo "Configuring build settings..." # URL safe branch name -BRANCH_NAME_SAFE="$(echo -n $BITBUCKET_BRANCH | sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}')" +BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME" | sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}')" # Trim the branch name if longer than BRANCH_NAME_LENGTH_LIMIT and append the md5sum to keep the branch name unique if (( "${#BRANCH_NAME_SAFE}" > "$BRANCH_NAME_LENGTH_LIMIT" )); then - BRANCH_NAME_SAFE="$(echo -n $BRANCH_NAME_SAFE | cut -c1-30)-$(echo -n $BRANCH_NAME_SAFE | md5sum | cut -c1-4)" + BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME_SAFE" | cut -c1-30)-$(echo -n "$BRANCH_NAME_SAFE" | md5sum | cut -c1-4)" fi -# Trim repo slug to 30 characters. If someone has a repo name longer than that, then no mercy for them. -export BITBUCKET_REPO_SLUG_SAFE="${BITBUCKET_REPO_SLUG:0:30}" +# Trim repo name to 30 characters. If someone has a repo name longer than that, then no mercy for them. +export REPO_NAME_SAFE="${REPO_NAME:0:30}" export BRANCH_NAME_SAFE -export REMOTE_BUILD_DIR="/home/ubuntu/builds/$BITBUCKET_REPO_SLUG_SAFE-$BRANCH_NAME_SAFE" -export COMPOSE_PROJECT_NAME="$BITBUCKET_REPO_SLUG_SAFE-$BRANCH_NAME_SAFE" +export REMOTE_BUILD_DIR="/home/ubuntu/builds/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" +export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" +export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" # Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with xip.io DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.xip.io}" sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config -# User ubuntu as a user by default +# Use ubuntu as the user by default DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config # NOTE: The length of any one label in the domain name is limited to between 1 and 63 octets. -export DOMAIN="$BRANCH_NAME_SAFE.$BITBUCKET_REPO_SLUG_SAFE.$DOCKSAL_HOST" +export DOMAIN="$BRANCH_NAME_SAFE.$REPO_NAME_SAFE.$DOCKSAL_HOST" echo "Configuring ssh access..." (umask 077 ; echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa) (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa) # Configure git -git config --global user.email "${CI_GIT_USER_EMAIL:-bitbucket-pipelines-agent@docksal.io}" -git config --global user.name "${CI_GIT_USER_NAME:-Bitbucket Pipelines Agent}" +git config --global user.email "${CI_GIT_USER_EMAIL:-ci@docksal.io}" +git config --global user.name "${CI_GIT_USER_NAME:-Docksal CI}" # Initialize a tunnel to the Docker Engine on DOCKSAL_HOST if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then diff --git a/base/bin/build-init b/base/bin/build-init index afded26..1f5ffd3 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -8,7 +8,7 @@ ssh docker-host whoami ssh docker-host "(cd $REMOTE_BUILD_DIR && fin rm -f) || true" ssh docker-host "sudo rm -rf $REMOTE_BUILD_DIR; mkdir -p $REMOTE_BUILD_DIR" # Checkout sources -ssh docker-host "cd $REMOTE_BUILD_DIR && git clone --branch="$BITBUCKET_BRANCH" --depth 50 git@bitbucket.org:$BITBUCKET_REPO_OWNER/$BITBUCKET_REPO_SLUG.git . && git reset --hard $BITBUCKET_COMMIT && ls -la" +ssh docker-host "cd $REMOTE_BUILD_DIR && git clone --branch="$BRANCH_NAME" --depth 50 git@bitbucket.org:$REPO_OWNER/$REPO_NAME.git . && git reset --hard $COMMIT_HASH && ls -la" # Configure sandbox settings echo "Configuring sandbox settings..." ssh docker-host "cd $REMOTE_BUILD_DIR && echo COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME | tee -a .docksal/docksal-local.env" From a317d82ecc26e93188d7460dfe7c0a2a4bce616a Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 27 Jun 2017 16:49:37 -0700 Subject: [PATCH 05/71] Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST Default to DOCKSAL_HOST if not set. --- base/bin/build-env | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/base/bin/build-env b/base/bin/build-env index 365112c..ceb29bb 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -52,8 +52,11 @@ sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config +# Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST. Default to DOCKSAL_HOST if not set. +# This is useful when working with CDNs/ELBs/WAFs/etc (when DOCKSAL_DOMAIN is different from the DOCKSAL_HOST). +DOCKSAL_DOMAIN="${DOCKSAL_DOMAIN:-$DOCKSAL_HOST}" # NOTE: The length of any one label in the domain name is limited to between 1 and 63 octets. -export DOMAIN="$BRANCH_NAME_SAFE.$REPO_NAME_SAFE.$DOCKSAL_HOST" +export DOMAIN="$BRANCH_NAME_SAFE.$REPO_NAME_SAFE.$DOCKSAL_DOMAIN" echo "Configuring ssh access..." (umask 077 ; echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa) From 93bc18d3a65d345c51854012425e06b68e3de1a3 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 27 Jun 2017 17:07:04 -0700 Subject: [PATCH 06/71] Make sure all build variables are exported --- base/bin/build-env | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index ceb29bb..e1920d8 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -2,24 +2,27 @@ # This script configures the pipelines build agent. +# Constants BRANCH_NAME_LENGTH_LIMIT=35 +echo "Configuring build settings..." + # Support for Bitbucket Pipelines if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then echo "Detected Bitbucket Pipelines build environment" - REPO_OWNER="$BITBUCKET_REPO_OWNER" - REPO_NAME="$BITBUCKET_REPO_SLUG" - BRANCH_NAME="$BITBUCKET_BRANCH" - COMMIT_HASH="$BITBUCKET_COMMIT" + export REPO_OWNER="$BITBUCKET_REPO_OWNER" + export REPO_NAME="$BITBUCKET_REPO_SLUG" + export BRANCH_NAME="$BITBUCKET_BRANCH" + export COMMIT_HASH="$BITBUCKET_COMMIT" fi # Support for CircleCI 2.0 if [[ "$CIRCLECI" != "" ]]; then echo "Detected CircleCI build environment" - REPO_OWNER="$CIRCLE_PROJECT_USERNAME" - REPO_NAME="$CIRCLE_PROJECT_REPONAME" - BRANCH_NAME="$CIRCLE_BRANCH" - COMMIT_HASH="$CIRCLE_SHA1" + export REPO_OWNER="$CIRCLE_PROJECT_USERNAME" + export REPO_NAME="$CIRCLE_PROJECT_REPONAME" + export BRANCH_NAME="$CIRCLE_BRANCH" + export COMMIT_HASH="$CIRCLE_SHA1" fi # For debug purposes these variables can be set manually. @@ -29,32 +32,31 @@ if [[ "$REPO_OWNER" == "" ]] || [[ "$REPO_NAME" == "" ]] || [[ "$BRANCH_NAME" == exit 1 fi -echo "Configuring build settings..." # URL safe branch name BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME" | sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}')" # Trim the branch name if longer than BRANCH_NAME_LENGTH_LIMIT and append the md5sum to keep the branch name unique if (( "${#BRANCH_NAME_SAFE}" > "$BRANCH_NAME_LENGTH_LIMIT" )); then BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME_SAFE" | cut -c1-30)-$(echo -n "$BRANCH_NAME_SAFE" | md5sum | cut -c1-4)" fi +export BRANCH_NAME_SAFE # Trim repo name to 30 characters. If someone has a repo name longer than that, then no mercy for them. export REPO_NAME_SAFE="${REPO_NAME:0:30}" -export BRANCH_NAME_SAFE export REMOTE_BUILD_DIR="/home/ubuntu/builds/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" # Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with xip.io -DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.xip.io}" +export DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.xip.io}" sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config # Use ubuntu as the user by default -DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" +export DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config # Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST. Default to DOCKSAL_HOST if not set. # This is useful when working with CDNs/ELBs/WAFs/etc (when DOCKSAL_DOMAIN is different from the DOCKSAL_HOST). -DOCKSAL_DOMAIN="${DOCKSAL_DOMAIN:-$DOCKSAL_HOST}" +export DOCKSAL_DOMAIN="${DOCKSAL_DOMAIN:-$DOCKSAL_HOST}" # NOTE: The length of any one label in the domain name is limited to between 1 and 63 octets. export DOMAIN="$BRANCH_NAME_SAFE.$REPO_NAME_SAFE.$DOCKSAL_DOMAIN" From 1bb5be93d0ac2225a86c81620ce3bfee749f31d2 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Tue, 27 Jun 2017 21:26:03 -0700 Subject: [PATCH 07/71] Check if ssh tunnel already exists Moved code into ssh_tunnel_init function --- base/bin/build-env | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index e1920d8..885b1bb 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -1,9 +1,33 @@ #!/usr/bin/env bash -# This script configures the pipelines build agent. +# Configures build environment variables + +# -------------------- Constants -------------------- # -# Constants BRANCH_NAME_LENGTH_LIMIT=35 +DOCKER_HOST_TUNNEL=localhost:2374 + +# -------------------- Functions -------------------- # + +ssh_tunnel_init () +{ + # Check if the tunnel is already active and return if so + ssh -O "check" docker-host >/dev/null 2>&1 + [[ $? == 0 ]] && return 0 + + echo "Setting up a secure tunnel to the Docker Engine on $DOCKSAL_HOST..." + # Black magic! Remote docker.sock access over SSH tunnel + # Credits: + # https://docs.docker.com/docker-for-aws/deploy/#connecting-via-ssh + # https://gist.github.com/scy/6781836#gistcomment-1559506 + ssh -fM -NL ${DOCKER_HOST_TUNNEL}:/var/run/docker.sock docker-host + + echo "Querying Docker Engine..." + docker --host ${DOCKER_HOST_TUNNEL} version + return $? +} + +# -------------------- Runtime -------------------- # echo "Configuring build settings..." @@ -69,16 +93,5 @@ git config --global user.email "${CI_GIT_USER_EMAIL:-ci@docksal.io}" git config --global user.name "${CI_GIT_USER_NAME:-Docksal CI}" # Initialize a tunnel to the Docker Engine on DOCKSAL_HOST -if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then - echo "Setting up a secure tunnel to the Docker Engine on $DOCKSAL_HOST..." - # Black magic! Remote docker.sock access over SSH tunnel - # Credits: - # https://docs.docker.com/docker-for-aws/deploy/#connecting-via-ssh - # https://gist.github.com/scy/6781836#gistcomment-1559506 - ssh -fM -NL localhost:2374:/var/run/docker.sock docker-host - - echo "Querying Docker Engine..." - docker --host localhost:2374 version - # Export local tunnel connection settings if it works - [[ $? == 0 ]] && export DOCKER_HOST="localhost:2374" -fi +# Export local tunnel connection settings if it works +[[ "$DOCKSAL_HOST_TUNNEL" != "" ]] && ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} From 4155330089f2398f6c5fdc1e6434f08c6e1ce520 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 28 Jun 2017 03:02:48 -0700 Subject: [PATCH 08/71] Rename variables GIT_REPO_OWNER GIT_REPO_NAME GIT_REPO_URL GIT_BRANCH_NAME GIT_COMMIT_HASH --- base/bin/build-env | 30 +++++++++++++++++------------- base/bin/build-init | 2 +- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 885b1bb..4387017 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -6,6 +6,8 @@ BRANCH_NAME_LENGTH_LIMIT=35 DOCKER_HOST_TUNNEL=localhost:2374 +GIT_USER_EMAIL=${GIT_USER_EMAIL:-ci@docksal.io} +GIT_USER_NAME=${GIT_USER_NAME:-Docksal CI} # -------------------- Functions -------------------- # @@ -34,30 +36,32 @@ echo "Configuring build settings..." # Support for Bitbucket Pipelines if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then echo "Detected Bitbucket Pipelines build environment" - export REPO_OWNER="$BITBUCKET_REPO_OWNER" - export REPO_NAME="$BITBUCKET_REPO_SLUG" - export BRANCH_NAME="$BITBUCKET_BRANCH" - export COMMIT_HASH="$BITBUCKET_COMMIT" + export GIT_REPO_OWNER="$BITBUCKET_REPO_OWNER" + export GIT_REPO_NAME="$BITBUCKET_REPO_SLUG" + export GIT_REPO_URL="git@bitbucket.org:$GIT_REPO_OWNER/$GIT_REPO_NAME.git" + export GIT_BRANCH_NAME="$BITBUCKET_BRANCH" + export GIT_COMMIT_HASH="$BITBUCKET_COMMIT" fi # Support for CircleCI 2.0 if [[ "$CIRCLECI" != "" ]]; then echo "Detected CircleCI build environment" - export REPO_OWNER="$CIRCLE_PROJECT_USERNAME" - export REPO_NAME="$CIRCLE_PROJECT_REPONAME" - export BRANCH_NAME="$CIRCLE_BRANCH" - export COMMIT_HASH="$CIRCLE_SHA1" + export GIT_REPO_OWNER="$CIRCLE_PROJECT_USERNAME" + export GIT_REPO_NAME="$CIRCLE_PROJECT_REPONAME" + export GIT_REPO_URL="$CIRCLE_REPOSITORY_URL" + export GIT_BRANCH_NAME="$CIRCLE_BRANCH" + export GIT_COMMIT_HASH="$CIRCLE_SHA1" fi # For debug purposes these variables can be set manually. # If they are empty here, then we cannot proceed. -if [[ "$REPO_OWNER" == "" ]] || [[ "$REPO_NAME" == "" ]] || [[ "$BRANCH_NAME" == "" ]]; then +if [[ "$GIT_REPO_URL" == "" ]] || [[ "$GIT_BRANCH_NAME" == "" ]] || [[ "$GIT_COMMIT_HASH" == "" ]]; then echo "Unsupported build environment! Quiting..." exit 1 fi # URL safe branch name -BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME" | sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}')" +BRANCH_NAME_SAFE="$(echo -n "$GIT_BRANCH_NAME" | sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}')" # Trim the branch name if longer than BRANCH_NAME_LENGTH_LIMIT and append the md5sum to keep the branch name unique if (( "${#BRANCH_NAME_SAFE}" > "$BRANCH_NAME_LENGTH_LIMIT" )); then BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME_SAFE" | cut -c1-30)-$(echo -n "$BRANCH_NAME_SAFE" | md5sum | cut -c1-4)" @@ -65,7 +69,7 @@ fi export BRANCH_NAME_SAFE # Trim repo name to 30 characters. If someone has a repo name longer than that, then no mercy for them. -export REPO_NAME_SAFE="${REPO_NAME:0:30}" +export REPO_NAME_SAFE="${GIT_REPO_NAME:0:30}" export REMOTE_BUILD_DIR="/home/ubuntu/builds/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" @@ -89,8 +93,8 @@ echo "Configuring ssh access..." (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa) # Configure git -git config --global user.email "${CI_GIT_USER_EMAIL:-ci@docksal.io}" -git config --global user.name "${CI_GIT_USER_NAME:-Docksal CI}" +git config --global user.email "$GIT_USER_EMAIL" +git config --global user.name "$GIT_USER_NAME" # Initialize a tunnel to the Docker Engine on DOCKSAL_HOST # Export local tunnel connection settings if it works diff --git a/base/bin/build-init b/base/bin/build-init index 1f5ffd3..56786d9 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -8,7 +8,7 @@ ssh docker-host whoami ssh docker-host "(cd $REMOTE_BUILD_DIR && fin rm -f) || true" ssh docker-host "sudo rm -rf $REMOTE_BUILD_DIR; mkdir -p $REMOTE_BUILD_DIR" # Checkout sources -ssh docker-host "cd $REMOTE_BUILD_DIR && git clone --branch="$BRANCH_NAME" --depth 50 git@bitbucket.org:$REPO_OWNER/$REPO_NAME.git . && git reset --hard $COMMIT_HASH && ls -la" +ssh docker-host "cd $REMOTE_BUILD_DIR && git clone --branch="$GIT_BRANCH_NAME" --depth 50 $GIT_REPO_URL . && git reset --hard $GIT_COMMIT_HASH && ls -la" # Configure sandbox settings echo "Configuring sandbox settings..." ssh docker-host "cd $REMOTE_BUILD_DIR && echo COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME | tee -a .docksal/docksal-local.env" From 93c17601d3ac7b5184898b2f6369fee3bb5acb7d Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 16 Aug 2017 11:03:14 -0700 Subject: [PATCH 09/71] README updates --- README.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1d8f7c7..57f9685 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,11 @@ URLs to sandbox environments can be found in the build logs and can also publish - `docksal/bitbucket-pipelines-agent:edge` - `docksal/bitbucket-pipelines-agent:edge-php` -## Configuration -### Global Pipelines variables +## Configuration -The agent expects the following required variables to be defined. +The following required variables should be configured at the Bitbucket organization level (this way all +project repos will have access to them). `DOCKSAL_HOST` or `DOCKSAL_HOST_IP` @@ -46,13 +46,31 @@ tutorial for details. `CI_SSH_KEY` -A second SSH keys (base64 encoded as well), which can be used for deployments and other remote operations run directly +A secondary SSH key (base64 encoded as well), which can be used for deployments and other remote operations run directly on the agent. E.g. cloning/pushing a repo, running commands over SSH on a remote deployment environment. -### Project level Pipelines variables +Other features and integrations are usually configured at the Bitbucket repo level. See below. + + +## Basic HTTP Auth + +Protect sandboxes from public access using Basic HTTP authentication. + +### Configuration -The following environment variables can be configured in the project's Pipelines settings: +Set the following environment variables at the repo level: + +- `HTTP_USER` +- `HTTP_PASS` + + +## Slack integration + +This integrations allows the agent to post messages to a given Slack channel. +It can be used for notification purposes when a build is started, completed, failed, etc. + +### Configuration `SLACK_WEBHOOK_URL` @@ -63,13 +81,11 @@ e.g. `SLACK_WEBHOOK_URL https://hooks.slack.com/services/XXXXXXXXX/XXXXXXXXX/XXx A public or private channel in Slack, e.g. `SLACK_CHANNEL #project-name-bots` -## Slack notifications - ### Usage `slack 'message' ['#channel'] ['webhook_url']` -Channel and webhook url can be passed via environment variables. See below. +Channel and webhook url can be passed via environment variables. See above. ### Limitations From 08458ba170cee7664b57f5afca1dd676e6cc36e9 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 16:59:17 -0700 Subject: [PATCH 10/71] Version updates - Alpine 3.6 - Docker 17.06.0-ce - Docker Compose 1.14.0 - Composer 1.5.1 - Drush 8.1.13 - Drupal Console 1.0.1 - WP CLI 1.3.0 --- base/Dockerfile | 17 +++++++---------- php/Dockerfile | 8 ++++---- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index bdd2879..36360c2 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.5 +FROM alpine:3.6 RUN apk add --update --no-cache \ bash \ @@ -19,18 +19,15 @@ RUN \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib -ARG DOCKER_VERSION=17.05.0-ce -ARG DOCKER_COMPOSE_VERSION=1.13.0 +ARG DOCKER_VERSION=17.06.0-ce +ARG DOCKER_COMPOSE_VERSION=1.14.0 RUN \ # Install docker cli - curl -L -O https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz && \ - tar -xzf docker-${DOCKER_VERSION}.tgz && \ - mv docker/docker /usr/local/bin && \ - rm -rf docker docker-${DOCKER_VERSION}.tgz && \ + curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz" && \ + tar zxf docker-$DOCKER_VERSION.tgz && mv docker/docker /usr/local/bin && rm -rf docker-$DOCKER_VERSION* && \ # Install docker-compose cli - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose && \ - # Make binaries executable - chmod -R +x /usr/local/bin + curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose && \ + chmod +x /usr/local/bin/docker-compose ARG AGENT_USER=agent ARG AGENT_HOME=/home/agent diff --git a/php/Dockerfile b/php/Dockerfile index bf205d4..f1b760d 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -16,10 +16,10 @@ RUN apk add --update --no-cache \ && rm -rf /var/cache/apk/* \ && ln -s /usr/bin/php7 /usr/bin/php -ENV COMPOSER_VERSION 1.3.0 -ENV DRUSH_VERSION 8.1.9 -ENV DRUPAL_CONSOLE_VERSION 1.0.0-rc15 -ENV WPCLI_VERSION 1.1.0 +ENV COMPOSER_VERSION=1.5.1 +ENV DRUSH_VERSION 8.1.13 +ENV DRUPAL_CONSOLE_VERSION 1.0.1 +ENV WPCLI_VERSION 1.3.0 RUN \ # Composer curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer && \ From 22df0c6bacfc587397612c6b20b32552592d06eb Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 17:07:01 -0700 Subject: [PATCH 11/71] php7-zlib Required by Drupal Console --- php/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/php/Dockerfile b/php/Dockerfile index f1b760d..92b7bd4 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -13,6 +13,7 @@ RUN apk add --update --no-cache \ php7-openssl \ php7-phar \ php7-xml \ + php7-zlib \ && rm -rf /var/cache/apk/* \ && ln -s /usr/bin/php7 /usr/bin/php From 76f016b507a2a26c17364d06ea0fcd1ccfa68be8 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 17:07:45 -0700 Subject: [PATCH 12/71] Makefile + basic bats tests + Travis CI --- .travis.yml | 20 +++++++++++++++++++ Makefile | 43 +++++++++++++++++++++++++++++++++++++++ tests/base.bats | 34 +++++++++++++++++++++++++++++++ tests/php.bats | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+) create mode 100644 .travis.yml create mode 100644 Makefile create mode 100755 tests/base.bats create mode 100755 tests/php.bats diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..8a88ff6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +language: generic + +services: + - docker + +env: + matrix: + - VERSION=base + - VERSION=php + +install: + - curl -fsSL get.docksal.io | sh + - fin version + - fin sysinfo + +script: + - make && make test + +after_failure: + - make logs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2e2640c --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +-include env_make + +VERSION ?= base + +REPO = docksal/bitbucket-pipelines-agent +NAME = bitbucket-pipelines-agent + +.PHONY: build test push shell run start stop logs clean release + +build: + fin docker build -t $(REPO):$(VERSION) $(VERSION) + +test: + IMAGE=$(REPO):$(VERSION) NAME=$(NAME) tests/$(VERSION).bats + +push: + fin docker push $(REPO):$(VERSION) + +shell: clean + fin docker run --rm --name $(NAME) -it $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) /bin/bash + +exec: + fin docker exec $(NAME) $(COMMAND) + +run: clean + fin docker run --rm --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) + +start: clean + fin docker run -d --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) top -b + +stop: + fin docker stop $(NAME) + +logs: + fin docker logs $(NAME) + +clean: + fin docker rm -f $(NAME) || true + +release: build + make push -e VERSION=$(VERSION) + +default: build diff --git a/tests/base.bats b/tests/base.bats new file mode 100755 index 0000000..9c7fa74 --- /dev/null +++ b/tests/base.bats @@ -0,0 +1,34 @@ +#!/usr/bin/env bats + +# Debugging +teardown() { + echo + echo "Last command status: $status" + echo "Last command output:" + echo "================================================================" + echo "$output" + echo "================================================================" +} + +# Global skip +# Uncomment below, then comment skip in the test you want to debug. When done, reverse. +#SKIP=1 + +@test "Check binaries" { + [[ $SKIP == 1 ]] && skip + + ### Setup ### + make start + + ### Tests ### + run make exec COMMAND="docker --version" + [[ "$status" == 0 ]] + echo "$output" | grep "Docker version" + + run make exec COMMAND="docker-compose --version" + [[ "$status" == 0 ]] + echo "$output" | grep "docker-compose version" + + ### Cleanup ### + make clean +} diff --git a/tests/php.bats b/tests/php.bats new file mode 100755 index 0000000..5544c27 --- /dev/null +++ b/tests/php.bats @@ -0,0 +1,53 @@ +#!/usr/bin/env bats + +# Debugging +teardown() { + echo + echo "Last command status: $status" + echo "Last command output:" + echo "================================================================" + echo "$output" + echo "================================================================" +} + +# Global skip +# Uncomment below, then comment skip in the test you want to debug. When done, reverse. +#SKIP=1 + +@test "Base tests" { + [[ $SKIP == 1 ]] && skip + + run bats tests/base.bats + [[ "$status" == 0 ]] +} + +@test "Check binaries" { + [[ $SKIP == 1 ]] && skip + + ### Setup ### + make start + + ### Tests ### + run make exec COMMAND="php --version" + [[ "$status" == 0 ]] + echo "$output" | grep "PHP" + + run make exec COMMAND="composer --version" + [[ "$status" == 0 ]] + echo "$output" | grep "Composer version" + + run make exec COMMAND="drush --version" + [[ "$status" == 0 ]] + echo "$output" | grep "Drush Version" + + run make exec COMMAND="drupal --version" + [[ "$status" == 0 ]] + echo "$output" | grep "Drupal Console Launcher" + + run make exec COMMAND="wp --version" + [[ "$status" == 0 ]] + echo "$output" | grep "WP-CLI" + + ### Cleanup ### + make clean +} From 6e320f615ee4452ab349e6e032ef88d7c62efec5 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 18:20:01 -0700 Subject: [PATCH 13/71] Added build-artifact-publish - build-artifact-publish - used to push build artifacts to S3 - minio client (mc) - updated tests --- base/Dockerfile | 5 ++++- base/bin/build-artifact-publish | 15 +++++++++++++++ tests/base.bats | 4 ++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100755 base/bin/build-artifact-publish diff --git a/base/Dockerfile b/base/Dockerfile index 36360c2..9798f07 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -27,7 +27,10 @@ RUN \ tar zxf docker-$DOCKER_VERSION.tgz && mv docker/docker /usr/local/bin && rm -rf docker-$DOCKER_VERSION* && \ # Install docker-compose cli curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose && \ - chmod +x /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose && \ + # Install minio client (mc) + curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && \ + chmod +x /usr/local/bin/mc ARG AGENT_USER=agent ARG AGENT_HOME=/home/agent diff --git a/base/bin/build-artifact-publish b/base/bin/build-artifact-publish new file mode 100755 index 0000000..3d91f25 --- /dev/null +++ b/base/bin/build-artifact-publish @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Publishes build artifacts +# +# Usage: build-artifact-publish + +# Initialize minio client (mc) configuration for S3 +mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} + +BUCKET="s3/${AWS_S3_BUCKET}" + +source=${1} +target="${BUCKET}/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/${GIT_BRANCH_NAME}-${GIT_COMMIT_HASH}" + +mc cp -r ${source} ${target} diff --git a/tests/base.bats b/tests/base.bats index 9c7fa74..dda52af 100755 --- a/tests/base.bats +++ b/tests/base.bats @@ -29,6 +29,10 @@ teardown() { [[ "$status" == 0 ]] echo "$output" | grep "docker-compose version" + run make exec COMMAND="mc --help" + [[ "$status" == 0 ]] + echo "$output" | grep "VERSION" + ### Cleanup ### make clean } From 069f6fb9892a4785e4e47f560c7a96e3877a4e40 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 18:20:34 -0700 Subject: [PATCH 14/71] .gitignore [ci skip] --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d143c2d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +env_make From 40578bc52354f98712f7be2da0d9c261fce3f14d Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 20:44:57 -0700 Subject: [PATCH 15/71] Alpine 3.6 compatibility fixes for php --- php/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/php/Dockerfile b/php/Dockerfile index 92b7bd4..286b3f3 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -12,10 +12,11 @@ RUN apk add --update --no-cache \ php7-mbstring \ php7-openssl \ php7-phar \ + php7-simplexml \ + php7-tokenizer \ php7-xml \ php7-zlib \ - && rm -rf /var/cache/apk/* \ - && ln -s /usr/bin/php7 /usr/bin/php + && rm -rf /var/cache/apk/* ENV COMPOSER_VERSION=1.5.1 ENV DRUSH_VERSION 8.1.13 From 7404292b609f8d491dfa065bf1eb44bf973f9d41 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 20:45:52 -0700 Subject: [PATCH 16/71] Convert indents to tabs in Dockerfiles --- base/Dockerfile | 56 ++++++++++++++++++++--------------------- php/Dockerfile | 66 ++++++++++++++++++++++++------------------------- 2 files changed, 61 insertions(+), 61 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index 9798f07..ae832d1 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,51 +1,51 @@ FROM alpine:3.6 RUN apk add --update --no-cache \ - bash \ - curl \ - git \ - jq \ - openssh \ - sudo \ - && rm -rf /var/cache/apk/* + bash \ + curl \ + git \ + jq \ + openssh \ + sudo \ + && rm -rf /var/cache/apk/* ARG GLIBC=2.25-r0 RUN \ - # Install glibc libraries needed for docker-compose - apk update && apk add --no-cache openssl ca-certificates && \ - curl -L https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub && \ - curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ - apk add --no-cache glibc-$GLIBC.apk && rm glibc-$GLIBC.apk && \ - ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ - ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib + # Install glibc libraries needed for docker-compose + apk update && apk add --no-cache openssl ca-certificates && \ + curl -L https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub && \ + curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ + apk add --no-cache glibc-$GLIBC.apk && rm glibc-$GLIBC.apk && \ + ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ + ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib ARG DOCKER_VERSION=17.06.0-ce ARG DOCKER_COMPOSE_VERSION=1.14.0 RUN \ - # Install docker cli - curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz" && \ - tar zxf docker-$DOCKER_VERSION.tgz && mv docker/docker /usr/local/bin && rm -rf docker-$DOCKER_VERSION* && \ - # Install docker-compose cli - curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose && \ - chmod +x /usr/local/bin/docker-compose && \ - # Install minio client (mc) - curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && \ - chmod +x /usr/local/bin/mc + # Install docker cli + curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz" && \ + tar zxf docker-$DOCKER_VERSION.tgz && mv docker/docker /usr/local/bin && rm -rf docker-$DOCKER_VERSION* && \ + # Install docker-compose cli + curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose && \ + chmod +x /usr/local/bin/docker-compose && \ + # Install minio client (mc) + curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && \ + chmod +x /usr/local/bin/mc ARG AGENT_USER=agent ARG AGENT_HOME=/home/agent RUN \ - # Create a non-root user with access to sudo - adduser -h $AGENT_HOME -s /bin/bash -D $AGENT_USER && \ - echo "$AGENT_USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + # Create a non-root user with access to sudo + adduser -h $AGENT_HOME -s /bin/bash -D $AGENT_USER && \ + echo "$AGENT_USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers # All further RUN commands will run as the "agent" user USER $AGENT_USER ENV HOME $AGENT_HOME RUN \ - mkdir -p $AGENT_HOME/.ssh && \ - mkdir -p $AGENT_HOME/build + mkdir -p $AGENT_HOME/.ssh && \ + mkdir -p $AGENT_HOME/build COPY bin /usr/local/bin COPY config/.ssh/config $AGENT_HOME/.ssh/config diff --git a/php/Dockerfile b/php/Dockerfile index 286b3f3..611a67f 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -4,35 +4,35 @@ FROM docksal/bitbucket-pipelines-agent:edge USER root RUN apk add --update --no-cache \ - php7 \ - php7-ctype \ - php7-curl \ - php7-dom \ - php7-json \ - php7-mbstring \ - php7-openssl \ - php7-phar \ - php7-simplexml \ - php7-tokenizer \ - php7-xml \ - php7-zlib \ - && rm -rf /var/cache/apk/* + php7 \ + php7-ctype \ + php7-curl \ + php7-dom \ + php7-json \ + php7-mbstring \ + php7-openssl \ + php7-phar \ + php7-simplexml \ + php7-tokenizer \ + php7-xml \ + php7-zlib \ + && rm -rf /var/cache/apk/* ENV COMPOSER_VERSION=1.5.1 ENV DRUSH_VERSION 8.1.13 ENV DRUPAL_CONSOLE_VERSION 1.0.1 ENV WPCLI_VERSION 1.3.0 RUN \ - # Composer - curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer && \ - # Drush 8 (default) - curl -sSL "https://github.com/drush-ops/drush/releases/download/${DRUSH_VERSION}/drush.phar" -o /usr/local/bin/drush && \ - # Drupal Console - curl -sSL "https://github.com/hechoendrupal/drupal-console-launcher/releases/download/${DRUPAL_CONSOLE_VERSION}/drupal.phar" -o /usr/local/bin/drupal && \ - # Install wp-cli - curl -sSL "https://github.com/wp-cli/wp-cli/releases/download/v${WPCLI_VERSION}/wp-cli-${WPCLI_VERSION}.phar" -o /usr/local/bin/wp && \ - # Make all binaries executable - chmod +x /usr/local/bin/* + # Composer + curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer && \ + # Drush 8 (default) + curl -sSL "https://github.com/drush-ops/drush/releases/download/${DRUSH_VERSION}/drush.phar" -o /usr/local/bin/drush && \ + # Drupal Console + curl -sSL "https://github.com/hechoendrupal/drupal-console-launcher/releases/download/${DRUPAL_CONSOLE_VERSION}/drupal.phar" -o /usr/local/bin/drupal && \ + # Install wp-cli + curl -sSL "https://github.com/wp-cli/wp-cli/releases/download/v${WPCLI_VERSION}/wp-cli-${WPCLI_VERSION}.phar" -o /usr/local/bin/wp && \ + # Make all binaries executable + chmod +x /usr/local/bin/* # Switch back to user ARG AGENT_USER=agent @@ -40,16 +40,16 @@ USER $AGENT_USER ENV PATH $PATH:$HOME/.composer/vendor/bin RUN \ - # Add composer bin directory to PATH - echo "\n"'PATH="$PATH:$HOME/.composer/vendor/bin"' >> $HOME/.profile && \ - # Drush modules - drush dl registry_rebuild --default-major=7 --destination=$HOME/.drush && \ - drush cc drush && \ - # Drupal Coder w/ a matching version of PHP_CodeSniffer - composer global require drupal/coder && \ - phpcs --config-set installed_paths $HOME/.composer/vendor/drupal/coder/coder_sniffer && \ - # Cleanup - composer clear-cache + # Add composer bin directory to PATH + echo "\n"'PATH="$PATH:$HOME/.composer/vendor/bin"' >> $HOME/.profile && \ + # Drush modules + drush dl registry_rebuild --default-major=7 --destination=$HOME/.drush && \ + drush cc drush && \ + # Drupal Coder w/ a matching version of PHP_CodeSniffer + composer global require drupal/coder && \ + phpcs --config-set installed_paths $HOME/.composer/vendor/drupal/coder/coder_sniffer && \ + # Cleanup + composer clear-cache COPY config/.drush $HOME/.drush # Fix permissions after COPY From 8cb77f2a9dbb6ab4b4350bfeb9cc39a8e89cdd2b Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 21:55:35 -0700 Subject: [PATCH 17/71] Renamed build-artifact-publish into build-acp and updated --- base/bin/build-acp | 35 +++++++++++++++++++++++++++++++++ base/bin/build-artifact-publish | 15 -------------- 2 files changed, 35 insertions(+), 15 deletions(-) create mode 100755 base/bin/build-acp delete mode 100755 base/bin/build-artifact-publish diff --git a/base/bin/build-acp b/base/bin/build-acp new file mode 100755 index 0000000..fe675f2 --- /dev/null +++ b/base/bin/build-acp @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Build Artifact Copy (build-acp) +# Copy build artifacts into S3 +# +# Usage: build-acp + +# Abort if anything failse +set -e + +# Initialize minio client (mc) configuration for S3 +mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} + +# If AWS_S3_BUCKET is set, derive BUCKET_PATH from repo name, branch and commit hash +if [[ "${AWS_S3_BUCKET}" != "" ]]; then + BUCKET_NAME="${AWS_S3_BUCKET}" + BUCKET_PATH="${GIT_REPO_NAME}/${GIT_BRANCH_NAME}-${GIT_COMMIT_HASH}" +# Otherwise use repo owner and repo name for BUCKET_NAME, git branch and commit hash for BUCKET_PATH +else + BUCKET_NAME="${GIT_REPO_OWNER}-${GIT_REPO_NAME}" + BUCKET_PATH="${GIT_BRANCH_NAME}-${GIT_COMMIT_HASH}" +fi + +URL="https://${BUCKET_NAME}.s3.amazonaws.com/${BUCKET_PATH}" + +source=${1} +target="s3/${BUCKET_NAME}/${BUCKET_PATH}" + +# Create bucket if it does not exist +mc mb "s3/${BUCKET_NAME}" + +# Upload artifact into the bucket +mc cp -r ${source} ${target} && + echo "Build artifacts upload completed:" && + echo "${URL}" diff --git a/base/bin/build-artifact-publish b/base/bin/build-artifact-publish deleted file mode 100755 index 3d91f25..0000000 --- a/base/bin/build-artifact-publish +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -# Publishes build artifacts -# -# Usage: build-artifact-publish - -# Initialize minio client (mc) configuration for S3 -mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} - -BUCKET="s3/${AWS_S3_BUCKET}" - -source=${1} -target="${BUCKET}/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/${GIT_BRANCH_NAME}-${GIT_COMMIT_HASH}" - -mc cp -r ${source} ${target} From 070aacffcfb2522a86cd2daa8e33ee22537b02d0 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 24 Aug 2017 23:03:09 -0700 Subject: [PATCH 18/71] Fixing build-env --- base/bin/build-env | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/base/bin/build-env b/base/bin/build-env index 4387017..293f4fa 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -93,9 +93,12 @@ echo "Configuring ssh access..." (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa) # Configure git +echo "Configuring git..." git config --global user.email "$GIT_USER_EMAIL" git config --global user.name "$GIT_USER_NAME" # Initialize a tunnel to the Docker Engine on DOCKSAL_HOST # Export local tunnel connection settings if it works -[[ "$DOCKSAL_HOST_TUNNEL" != "" ]] && ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} +if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then + ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} +fi From d0417ec6de717bab0a3e4c1ea22a7c370aa3f6d3 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 25 Aug 2017 10:18:01 -0700 Subject: [PATCH 19/71] Post the URL to artifacts via Bitbucket build status API --- base/bin/build-acp | 45 ++++++++++++++++++++++++++------------------- base/bin/build-env | 28 ++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index fe675f2..c5ff2bf 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -9,27 +9,34 @@ set -e # Initialize minio client (mc) configuration for S3 +# AWS API keys must be set in the build settings mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} -# If AWS_S3_BUCKET is set, derive BUCKET_PATH from repo name, branch and commit hash -if [[ "${AWS_S3_BUCKET}" != "" ]]; then - BUCKET_NAME="${AWS_S3_BUCKET}" - BUCKET_PATH="${GIT_REPO_NAME}/${GIT_BRANCH_NAME}-${GIT_COMMIT_HASH}" -# Otherwise use repo owner and repo name for BUCKET_NAME, git branch and commit hash for BUCKET_PATH -else - BUCKET_NAME="${GIT_REPO_OWNER}-${GIT_REPO_NAME}" - BUCKET_PATH="${GIT_BRANCH_NAME}-${GIT_COMMIT_HASH}" -fi - -URL="https://${BUCKET_NAME}.s3.amazonaws.com/${BUCKET_PATH}" - source=${1} -target="s3/${BUCKET_NAME}/${BUCKET_PATH}" +destination="s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" # Create bucket if it does not exist -mc mb "s3/${BUCKET_NAME}" - -# Upload artifact into the bucket -mc cp -r ${source} ${target} && - echo "Build artifacts upload completed:" && - echo "${URL}" +mc mb "s3/${ARTIFACTS_BUCKET_NAME}" +# Upload artifact into the bucket and get the list of files at the destination +mc cp -r ${source} ${destination} +# Set read-only permissions on BUCKET_PATH +# Access is possible only if BUCKET_PATH is known (parent bucket stays private => XML listing of bucket objects disabled) +mc policy download "s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" + +echo "Build artifacts upload completed:" +echo "${ARTIFACTS_URL}" +echo "Files:" +mc ls ${destination} + +# Post artifacts to Bitbucket build status API +if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then + echo "Posting artifacts URL to Bitbucket..." + + BUILD_STATUS_URL="${ARTIFACTS_URL}" + BUILD_STATUS_PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Build artifacts\", \"url\": \"${BUILD_STATUS_URL}\"}" + + BITBUCKET_API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" + curl -sS -H "Content-Type: application/json" -X POST --user "${BITBUCKET_API_AUTH}" -d "${BUILD_STATUS_PAYLOAD}" "${BITBUCKET_API_URL}" >/dev/null +else + echo "This command only works in Bitbucket Pipelines" +fi diff --git a/base/bin/build-env b/base/bin/build-env index 293f4fa..f7ce3f6 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -29,6 +29,29 @@ ssh_tunnel_init () return $? } +# Set environment variables for artifact storage functionality +artifact_storage_env () +{ + # Single bucket mode (all repos are stored in a single bucket) + # If AWS_S3_BUCKET is set, derive BUCKET_PATH from repo name, branch and commit hash + if [[ "${AWS_S3_BUCKET}" != "" ]]; then + export ARTIFACTS_BUCKET_NAME="${AWS_S3_BUCKET}" + export ARTIFACTS_BUCKET_PATH="${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}" + # Multi-bucket module (each repo gets its own bucket) + # Otherwise use repo owner and repo name for BUCKET_NAME, git branch and commit hash for BUCKET_PATH + else + # Using hyphen as a delimiter in the bucket name + # With the dot delimiter in BUCKET_NAME Amazon's wildcard cert (*.s3.amazonaws.com) does not work. + # E.g. https://my.bucket.name.s3.amazonaws.com/ throws an invalid cert warning. + # URLs like https://s3.amazonaws.com/$BUCKET_NAME works in either case (with hyphens or dots in BUCKET_NAME) + export ARTIFACTS_BUCKET_NAME="${REPO_NAME_SAFE}-${GIT_REPO_OWNER}" + export ARTIFACTS_BUCKET_PATH="${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}" + fi + + # Expecting index.html as the default entry point into the bucket/path + export ARTIFACTS_URL="https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com/${ARTIFACTS_BUCKET_PATH}/index.html" +} + # -------------------- Runtime -------------------- # echo "Configuring build settings..." @@ -99,6 +122,11 @@ git config --global user.name "$GIT_USER_NAME" # Initialize a tunnel to the Docker Engine on DOCKSAL_HOST # Export local tunnel connection settings if it works +# Using full if form instead of the short one here, otherwise builds will fail, when the condition below is false if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} fi + +# Artifact storage +echo "Configuring artifact storage settings..." +artifact_storage_env From e6b22db3d0a436673bbd3127b29c53800a273be7 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 25 Aug 2017 16:02:14 -0700 Subject: [PATCH 20/71] Refactored build-env - split into functions --- base/bin/build-env | 165 +++++++++++++++++++++++++++------------------ 1 file changed, 98 insertions(+), 67 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index f7ce3f6..c1e8042 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -1,6 +1,12 @@ #!/usr/bin/env bash # Configures build environment variables +# This file should be sources at the beginning of a build +# +# Usage source build-env + +# Abort if anything fails +set -e # -------------------- Constants -------------------- # @@ -11,6 +17,62 @@ GIT_USER_NAME=${GIT_USER_NAME:-Docksal CI} # -------------------- Functions -------------------- # +# Initial build environment configuration +build_env () +{ + # Support for Bitbucket Pipelines + if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then + echo "Detected Bitbucket Pipelines build environment" + export GIT_REPO_OWNER="$BITBUCKET_REPO_OWNER" + export GIT_REPO_NAME="$BITBUCKET_REPO_SLUG" + export GIT_REPO_URL="git@bitbucket.org:$GIT_REPO_OWNER/$GIT_REPO_NAME.git" + export GIT_BRANCH_NAME="$BITBUCKET_BRANCH" + export GIT_COMMIT_HASH="$BITBUCKET_COMMIT" + fi + + # Support for CircleCI 2.0 + if [[ "$CIRCLECI" != "" ]]; then + echo "Detected CircleCI build environment" + export GIT_REPO_OWNER="$CIRCLE_PROJECT_USERNAME" + export GIT_REPO_NAME="$CIRCLE_PROJECT_REPONAME" + export GIT_REPO_URL="$CIRCLE_REPOSITORY_URL" + export GIT_BRANCH_NAME="$CIRCLE_BRANCH" + export GIT_COMMIT_HASH="$CIRCLE_SHA1" + fi + + # For debug purposes these variables can be set manually. + # If they are empty here, then we cannot proceed. + if [[ "$GIT_REPO_URL" == "" ]] || [[ "$GIT_BRANCH_NAME" == "" ]] || [[ "$GIT_COMMIT_HASH" == "" ]]; then + echo "Unsupported build environment! Quiting..." + exit 1 + fi + + # URL safe branch name + BRANCH_NAME_SAFE="$(echo -n "$GIT_BRANCH_NAME" | sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}')" + # Trim the branch name if longer than BRANCH_NAME_LENGTH_LIMIT and append the md5sum to keep the branch name unique + if (( "${#BRANCH_NAME_SAFE}" > "$BRANCH_NAME_LENGTH_LIMIT" )); then + BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME_SAFE" | cut -c1-30)-$(echo -n "$BRANCH_NAME_SAFE" | md5sum | cut -c1-4)" + fi + export BRANCH_NAME_SAFE + + # Trim repo name to 30 characters. If someone has a repo name longer than that, then no mercy for them. + export REPO_NAME_SAFE="${GIT_REPO_NAME:0:30}" + # Short version of GIT_COMMIT_HASH + export COMMIT_HASH_SHORT="${GIT_COMMIT_HASH:0:7}" +} + +ssh_init () +{ + (umask 077 ; echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa) + (umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa) +} + +git_env () +{ + git config --global user.email "$GIT_USER_EMAIL" + git config --global user.name "$GIT_USER_NAME" +} + ssh_tunnel_init () { # Check if the tunnel is already active and return if so @@ -29,6 +91,34 @@ ssh_tunnel_init () return $? } +sandbox_server_env () +{ + export REMOTE_BUILD_DIR="/home/ubuntu/builds/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" + export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" + export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" + + # Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with xip.io + export DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.xip.io}" + sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config + + # Use ubuntu as the user by default + export DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" + sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config + + # Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST. Default to DOCKSAL_HOST if not set. + # This is useful when working with CDNs/ELBs/WAFs/etc (when DOCKSAL_DOMAIN is different from the DOCKSAL_HOST). + export DOCKSAL_DOMAIN="${DOCKSAL_DOMAIN:-$DOCKSAL_HOST}" + # NOTE: The length of any one label in the domain name is limited to between 1 and 63 octets. + export DOMAIN="$BRANCH_NAME_SAFE.$REPO_NAME_SAFE.$DOCKSAL_DOMAIN" + + # Initialize a tunnel to the Docker Engine on DOCKSAL_HOST + # Export local tunnel connection settings if it works + # Using full if form instead of the short one here, otherwise builds will fail, when the condition below is false + if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then + ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} + fi +} + # Set environment variables for artifact storage functionality artifact_storage_env () { @@ -55,77 +145,18 @@ artifact_storage_env () # -------------------- Runtime -------------------- # echo "Configuring build settings..." - -# Support for Bitbucket Pipelines -if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then - echo "Detected Bitbucket Pipelines build environment" - export GIT_REPO_OWNER="$BITBUCKET_REPO_OWNER" - export GIT_REPO_NAME="$BITBUCKET_REPO_SLUG" - export GIT_REPO_URL="git@bitbucket.org:$GIT_REPO_OWNER/$GIT_REPO_NAME.git" - export GIT_BRANCH_NAME="$BITBUCKET_BRANCH" - export GIT_COMMIT_HASH="$BITBUCKET_COMMIT" -fi - -# Support for CircleCI 2.0 -if [[ "$CIRCLECI" != "" ]]; then - echo "Detected CircleCI build environment" - export GIT_REPO_OWNER="$CIRCLE_PROJECT_USERNAME" - export GIT_REPO_NAME="$CIRCLE_PROJECT_REPONAME" - export GIT_REPO_URL="$CIRCLE_REPOSITORY_URL" - export GIT_BRANCH_NAME="$CIRCLE_BRANCH" - export GIT_COMMIT_HASH="$CIRCLE_SHA1" -fi - -# For debug purposes these variables can be set manually. -# If they are empty here, then we cannot proceed. -if [[ "$GIT_REPO_URL" == "" ]] || [[ "$GIT_BRANCH_NAME" == "" ]] || [[ "$GIT_COMMIT_HASH" == "" ]]; then - echo "Unsupported build environment! Quiting..." - exit 1 -fi - -# URL safe branch name -BRANCH_NAME_SAFE="$(echo -n "$GIT_BRANCH_NAME" | sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}')" -# Trim the branch name if longer than BRANCH_NAME_LENGTH_LIMIT and append the md5sum to keep the branch name unique -if (( "${#BRANCH_NAME_SAFE}" > "$BRANCH_NAME_LENGTH_LIMIT" )); then - BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME_SAFE" | cut -c1-30)-$(echo -n "$BRANCH_NAME_SAFE" | md5sum | cut -c1-4)" -fi -export BRANCH_NAME_SAFE - -# Trim repo name to 30 characters. If someone has a repo name longer than that, then no mercy for them. -export REPO_NAME_SAFE="${GIT_REPO_NAME:0:30}" -export REMOTE_BUILD_DIR="/home/ubuntu/builds/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" -export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" -export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" - -# Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with xip.io -export DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.xip.io}" -sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config - -# Use ubuntu as the user by default -export DOCKSAL_HOST_USER="${DOCKSAL_HOST_USER:-ubuntu}" -sed -i "s/User DOCKSAL_HOST_USER/User $DOCKSAL_HOST_USER/g" $HOME/.ssh/config - -# Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST. Default to DOCKSAL_HOST if not set. -# This is useful when working with CDNs/ELBs/WAFs/etc (when DOCKSAL_DOMAIN is different from the DOCKSAL_HOST). -export DOCKSAL_DOMAIN="${DOCKSAL_DOMAIN:-$DOCKSAL_HOST}" -# NOTE: The length of any one label in the domain name is limited to between 1 and 63 octets. -export DOMAIN="$BRANCH_NAME_SAFE.$REPO_NAME_SAFE.$DOCKSAL_DOMAIN" +build_env echo "Configuring ssh access..." -(umask 077 ; echo "$CI_SSH_KEY" | base64 -d > $HOME/.ssh/id_rsa) -(umask 077 ; echo "$DOCKSAL_HOST_SSH_KEY" | base64 -d > $HOME/.ssh/docksal_host_id_rsa) +ssh_init # Configure git -echo "Configuring git..." -git config --global user.email "$GIT_USER_EMAIL" -git config --global user.name "$GIT_USER_NAME" - -# Initialize a tunnel to the Docker Engine on DOCKSAL_HOST -# Export local tunnel connection settings if it works -# Using full if form instead of the short one here, otherwise builds will fail, when the condition below is false -if [[ "$DOCKSAL_HOST_TUNNEL" != "" ]]; then - ssh_tunnel_init && export DOCKER_HOST=${DOCKER_HOST_TUNNEL} -fi +echo "Configuring git settings..." +git_env + +# Sandbox server settings +echo "Configuring sandbox server settings..." +sandbox_server_env # Artifact storage echo "Configuring artifact storage settings..." From 34ce569ec90186ed03aa0afcb6a5e683aa50b350 Mon Sep 17 00:00:00 2001 From: Chris Zietlow Date: Wed, 20 Sep 2017 13:50:41 -0400 Subject: [PATCH 21/71] Install the xmlwriter module in the php pipeline container. --- php/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/php/Dockerfile b/php/Dockerfile index 611a67f..dfd182e 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -15,6 +15,7 @@ RUN apk add --update --no-cache \ php7-simplexml \ php7-tokenizer \ php7-xml \ + php7-xmlwriter \ php7-zlib \ && rm -rf /var/cache/apk/* From 9ef92ec58f1eb7405aabcc540905af3c68bd264e Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 22 Sep 2017 11:17:11 -0700 Subject: [PATCH 22/71] Included php7-xmlreader extension --- php/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/php/Dockerfile b/php/Dockerfile index dfd182e..af5b394 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -15,6 +15,7 @@ RUN apk add --update --no-cache \ php7-simplexml \ php7-tokenizer \ php7-xml \ + php7-xmlreader \ php7-xmlwriter \ php7-zlib \ && rm -rf /var/cache/apk/* From e7c569a4a0b747353d7c488f557bf4344d5c6136 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 16 Oct 2017 15:51:51 -0700 Subject: [PATCH 23/71] Renamed AWS_S3_BUCKET to ARTIFACTS_BUCKET_NAME --- base/bin/build-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index c1e8042..0b4ef4d 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -124,8 +124,8 @@ artifact_storage_env () { # Single bucket mode (all repos are stored in a single bucket) # If AWS_S3_BUCKET is set, derive BUCKET_PATH from repo name, branch and commit hash - if [[ "${AWS_S3_BUCKET}" != "" ]]; then - export ARTIFACTS_BUCKET_NAME="${AWS_S3_BUCKET}" + if [[ "${ARTIFACTS_BUCKET_NAME}" != "" ]]; then + export ARTIFACTS_BUCKET_NAME export ARTIFACTS_BUCKET_PATH="${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}" # Multi-bucket module (each repo gets its own bucket) # Otherwise use repo owner and repo name for BUCKET_NAME, git branch and commit hash for BUCKET_PATH From 863f99c435ccad1973ac8476b919ea76151209ab Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 16 Oct 2017 15:52:52 -0700 Subject: [PATCH 24/71] Made ARTIFACTS_BASE_URL configurable If not set, it will be derived from the ARTIFACTS_BUCKET_NAME --- base/bin/build-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/base/bin/build-env b/base/bin/build-env index 0b4ef4d..f726abd 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -138,8 +138,10 @@ artifact_storage_env () export ARTIFACTS_BUCKET_PATH="${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}" fi + # Derive artifacts base URL from the bucket name, unless ARTIFACTS_BASE_URL has been already defined + export ARTIFACTS_BASE_URL=${ARTIFACTS_BASE_URL:-"https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com"} # Expecting index.html as the default entry point into the bucket/path - export ARTIFACTS_URL="https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com/${ARTIFACTS_BUCKET_PATH}/index.html" + export ARTIFACTS_URL="${ARTIFACTS_BASE_URL}/${ARTIFACTS_BUCKET_PATH}/index.html" } # -------------------- Runtime -------------------- # From 5ded68a1371f4e5efcc8d393f75e5c1570d142bb Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 16 Oct 2017 17:05:30 -0700 Subject: [PATCH 25/71] Simplified artifact variable configuration - Got rid of the multi-bucket mode (unlikely to be used by anyone) - Allowed ARTIFACTS_BUCKET_PATH, ARTIFACTS_BASE_URL and ARTIFACTS_URL to be overridden --- base/bin/build-env | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index f726abd..2926e86 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -122,26 +122,18 @@ sandbox_server_env () # Set environment variables for artifact storage functionality artifact_storage_env () { - # Single bucket mode (all repos are stored in a single bucket) - # If AWS_S3_BUCKET is set, derive BUCKET_PATH from repo name, branch and commit hash - if [[ "${ARTIFACTS_BUCKET_NAME}" != "" ]]; then - export ARTIFACTS_BUCKET_NAME - export ARTIFACTS_BUCKET_PATH="${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}" - # Multi-bucket module (each repo gets its own bucket) - # Otherwise use repo owner and repo name for BUCKET_NAME, git branch and commit hash for BUCKET_PATH - else - # Using hyphen as a delimiter in the bucket name - # With the dot delimiter in BUCKET_NAME Amazon's wildcard cert (*.s3.amazonaws.com) does not work. - # E.g. https://my.bucket.name.s3.amazonaws.com/ throws an invalid cert warning. - # URLs like https://s3.amazonaws.com/$BUCKET_NAME works in either case (with hyphens or dots in BUCKET_NAME) - export ARTIFACTS_BUCKET_NAME="${REPO_NAME_SAFE}-${GIT_REPO_OWNER}" - export ARTIFACTS_BUCKET_PATH="${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}" + if [[ "$ARTIFACTS_BUCKET_NAME" == "" ]]; then + echo "Cannot upload artifacts. ARTIFACTS_BUCKET_NAME environment variable is not set." + exit fi - # Derive artifacts base URL from the bucket name, unless ARTIFACTS_BASE_URL has been already defined + # Configure ARTIFACTS_BUCKET_PATH, unless it has been already defined + export ARTIFACTS_BUCKET_PATH=${ARTIFACTS_BUCKET_PATH:-"${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}"} + # Configure ARTIFACTS_BASE_URL, unless it has been already defined export ARTIFACTS_BASE_URL=${ARTIFACTS_BASE_URL:-"https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com"} - # Expecting index.html as the default entry point into the bucket/path - export ARTIFACTS_URL="${ARTIFACTS_BASE_URL}/${ARTIFACTS_BUCKET_PATH}/index.html" + # Configure ARTIFACTS_URL, unless it has been already defined + # By default expecting index.html as the default entry point into the bucket/path + export ARTIFACTS_URL=${ARTIFACTS_URL:-"${ARTIFACTS_BASE_URL}/${ARTIFACTS_BUCKET_PATH}/index.html"} } # -------------------- Runtime -------------------- # From e0a0acf372c5ad5e18eb2f9c49c700dcb4f81f3e Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 16 Oct 2017 18:15:22 -0700 Subject: [PATCH 26/71] Moved artifact storage settings into build-acp This makes it easier to override ARTIFACTS_* variables at runtime --- base/bin/build-acp | 22 +++++++++++++++++++--- base/bin/build-env | 25 ++----------------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index c5ff2bf..ee78ee4 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -8,13 +8,29 @@ # Abort if anything failse set -e -# Initialize minio client (mc) configuration for S3 -# AWS API keys must be set in the build settings -mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} +if [[ "${AWS_ACCESS_KEY_ID}" == "" ]] || [[ "${AWS_SECRET_ACCESS_KEY}" == "" ]] || [[ "${ARTIFACTS_BUCKET_NAME}" == "" ]]; then + echo "Cannot upload artifacts." + echo "Please make sure AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and ARTIFACTS_BUCKET_NAME environment variables are configured." + exit 1 +fi + +# Configure ARTIFACTS_BASE_URL, unless it has been already defined +ARTIFACTS_BASE_URL=${ARTIFACTS_BASE_URL:-"https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com"} +# Configure ARTIFACTS_BUCKET_PATH, unless it has been already defined +ARTIFACTS_BUCKET_PATH=${ARTIFACTS_BUCKET_PATH:-"${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}"} +# Configure ARTIFACTS_URL, unless it has been already defined +# By default expecting index.html as the default entry point into the bucket/path +ARTIFACTS_URL=${ARTIFACTS_URL:-"${ARTIFACTS_BASE_URL}/${ARTIFACTS_BUCKET_PATH}/index.html"} +# Sync source locally source=${1} +# Sync destination on S3 destination="s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" +# Initialize minio client (mc) configuration for S3 +# AWS API keys must be set in the build settings +mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} + # Create bucket if it does not exist mc mb "s3/${ARTIFACTS_BUCKET_NAME}" # Upload artifact into the bucket and get the list of files at the destination diff --git a/base/bin/build-env b/base/bin/build-env index 2926e86..73de98a 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -5,8 +5,8 @@ # # Usage source build-env -# Abort if anything fails -set -e +set -e # Abort if anything fails +#set -x # Echo commands # -------------------- Constants -------------------- # @@ -119,23 +119,6 @@ sandbox_server_env () fi } -# Set environment variables for artifact storage functionality -artifact_storage_env () -{ - if [[ "$ARTIFACTS_BUCKET_NAME" == "" ]]; then - echo "Cannot upload artifacts. ARTIFACTS_BUCKET_NAME environment variable is not set." - exit - fi - - # Configure ARTIFACTS_BUCKET_PATH, unless it has been already defined - export ARTIFACTS_BUCKET_PATH=${ARTIFACTS_BUCKET_PATH:-"${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}"} - # Configure ARTIFACTS_BASE_URL, unless it has been already defined - export ARTIFACTS_BASE_URL=${ARTIFACTS_BASE_URL:-"https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com"} - # Configure ARTIFACTS_URL, unless it has been already defined - # By default expecting index.html as the default entry point into the bucket/path - export ARTIFACTS_URL=${ARTIFACTS_URL:-"${ARTIFACTS_BASE_URL}/${ARTIFACTS_BUCKET_PATH}/index.html"} -} - # -------------------- Runtime -------------------- # echo "Configuring build settings..." @@ -151,7 +134,3 @@ git_env # Sandbox server settings echo "Configuring sandbox server settings..." sandbox_server_env - -# Artifact storage -echo "Configuring artifact storage settings..." -artifact_storage_env From 79def68632b93561be0d196ff4f90dc4476aab58 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 16 Oct 2017 18:16:11 -0700 Subject: [PATCH 27/71] Added a check for non-empty source Outputting variables when empty source (this can be used for debugging) --- base/bin/build-acp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/base/bin/build-acp b/base/bin/build-acp index ee78ee4..82613ae 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -27,6 +27,17 @@ source=${1} # Sync destination on S3 destination="s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" +if [[ "${source}" == "" ]]; then + echo "No source provided." + echo "Configuration: " + echo "source=${source}" + echo "destination=${destination}" + echo "ARTIFACTS_BASE_URL=${ARTIFACTS_BASE_URL}" + echo "ARTIFACTS_BUCKET_PATH=${ARTIFACTS_BUCKET_PATH}" + echo "ARTIFACTS_URL=${ARTIFACTS_URL}" + exit +fi + # Initialize minio client (mc) configuration for S3 # AWS API keys must be set in the build settings mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} From 67b9943c15f53409f0755f083508426ae23280b6 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 16 Oct 2017 22:04:45 -0700 Subject: [PATCH 28/71] Use mc mirror instead of mc cp --- base/bin/build-acp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index 82613ae..c0cb2d1 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -44,8 +44,8 @@ mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET # Create bucket if it does not exist mc mb "s3/${ARTIFACTS_BUCKET_NAME}" -# Upload artifact into the bucket and get the list of files at the destination -mc cp -r ${source} ${destination} +# Upload artifact into the bucket +mc mirror --force --remove ${source} ${destination} # Set read-only permissions on BUCKET_PATH # Access is possible only if BUCKET_PATH is known (parent bucket stays private => XML listing of bucket objects disabled) mc policy download "s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" From f6f7bd6a8de570e0555b9a86408e789a447d7d6a Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 16 Oct 2017 23:08:21 -0700 Subject: [PATCH 29/71] Added a note about mc cp vs mc mirror in a build environment --- base/bin/build-acp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/bin/build-acp b/base/bin/build-acp index c0cb2d1..740559e 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -45,6 +45,8 @@ mc config host add s3 https://s3.amazonaws.com ${AWS_ACCESS_KEY_ID} ${AWS_SECRET # Create bucket if it does not exist mc mb "s3/${ARTIFACTS_BUCKET_NAME}" # Upload artifact into the bucket +# TODO: figure out how to skip timestemp checking. Maybe use aws-cli instead on the minio client? +# Looks like mirror does not have any benefit here over cp. Files in build have newer timestamps and are always copied. mc mirror --force --remove ${source} ${destination} # Set read-only permissions on BUCKET_PATH # Access is possible only if BUCKET_PATH is known (parent bucket stays private => XML listing of bucket objects disabled) From fbd6b38a0c7e5bea9a2a3c4d1b33f320cda045d4 Mon Sep 17 00:00:00 2001 From: Chris Zietlow Date: Fri, 15 Dec 2017 13:53:46 -0500 Subject: [PATCH 30/71] fixes #9 by adding support for gtlab CI. --- base/bin/build-env | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/base/bin/build-env b/base/bin/build-env index 73de98a..a7a8181 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -40,6 +40,16 @@ build_env () export GIT_COMMIT_HASH="$CIRCLE_SHA1" fi + # Support for GitLab 9.0+ + if [[ "$GITLAB_CI" != "" ]]; then + echo "Detected GitLabCI build environment" + export GIT_REPO_OWNER="$CI_PROJECT_NAMESPACE" + export GIT_REPO_NAME="$CI_PROJECT_NAME" + export GIT_REPO_URL="$CI_REPOSITORY_URL" + export GIT_BRANCH_NAME="$CI_COMMIT_REF_NAME" + export GIT_COMMIT_HASH="$CI_COMMIT_SHA" + fi + # For debug purposes these variables can be set manually. # If they are empty here, then we cannot proceed. if [[ "$GIT_REPO_URL" == "" ]] || [[ "$GIT_BRANCH_NAME" == "" ]] || [[ "$GIT_COMMIT_HASH" == "" ]]; then From 3dace31fdb1dee93a96b14340764883fa03aec05 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 20 Dec 2017 20:49:41 -0800 Subject: [PATCH 31/71] Added rsync Closes #8 --- base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/base/Dockerfile b/base/Dockerfile index ae832d1..9a6c3e7 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -6,6 +6,7 @@ RUN apk add --update --no-cache \ git \ jq \ openssh \ + rsync \ sudo \ && rm -rf /var/cache/apk/* From 9f995b1506dda16358d9dfb475f214b6d4aab4b5 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 20 Dec 2017 20:50:37 -0800 Subject: [PATCH 32/71] Bump versions docker 17.09.01 docker-compose 1.18.0 --- base/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index 9a6c3e7..0ea71aa 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -20,8 +20,8 @@ RUN \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib -ARG DOCKER_VERSION=17.06.0-ce -ARG DOCKER_COMPOSE_VERSION=1.14.0 +ARG DOCKER_VERSION=17.09.1-ce +ARG DOCKER_COMPOSE_VERSION=1.18.0 RUN \ # Install docker cli curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz" && \ From 36cacf69a449286568d2ac682c9b4e59dd4ae88f Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 20 Dec 2017 20:51:55 -0800 Subject: [PATCH 33/71] build-acp: allow specifying a subpath within the destination path --- base/bin/build-acp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/bin/build-acp b/base/bin/build-acp index 740559e..d76bd61 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -26,6 +26,8 @@ ARTIFACTS_URL=${ARTIFACTS_URL:-"${ARTIFACTS_BASE_URL}/${ARTIFACTS_BUCKET_PATH}/i source=${1} # Sync destination on S3 destination="s3/${ARTIFACTS_BUCKET_NAME}/${ARTIFACTS_BUCKET_PATH}" +# Allow specifying a subpath within the destination path +[[ "${2}" != "" ]] && destination="${destination}/${2}" if [[ "${source}" == "" ]]; then echo "No source provided." From a7167a9aa15672cf74e5fd15a4fcd1417ad51441 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 20 Dec 2017 20:52:43 -0800 Subject: [PATCH 34/71] build-acp: check BITBUCKET_API_AUTH is set --- base/bin/build-acp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index d76bd61..6dd036d 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -60,7 +60,7 @@ echo "Files:" mc ls ${destination} # Post artifacts to Bitbucket build status API -if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then +if [[ "${BITBUCKET_REPO_SLUG}" != "" ]] && [[ "${BITBUCKET_API_AUTH}" != "" ]]; then echo "Posting artifacts URL to Bitbucket..." BUILD_STATUS_URL="${ARTIFACTS_URL}" From a8af18615cb39331a2e1de7ee73e83faad8c8003 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 20 Dec 2017 21:03:39 -0800 Subject: [PATCH 35/71] Updated README and added examples --- README.md | 142 ++++++++++++++++-- bitbucket-pipelines.yml | 15 -- examples/.circleci/config.yml | 28 ++++ .../bitbucket-pipelines.yml | 11 ++ 4 files changed, 168 insertions(+), 28 deletions(-) delete mode 100644 bitbucket-pipelines.yml create mode 100644 examples/.circleci/config.yml create mode 100644 examples/bitbucket-pipelines/bitbucket-pipelines.yml diff --git a/README.md b/README.md index 57f9685..940a53d 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,44 @@ -# Bitbucket Pipelines Agent Docker image for Docksal CI +# CI agent Docker image for Docksal Sandboxes -A thin agent used to provision Docksal powered sandboxes on a remote Docker host. +A thin agent used to provision Docksal Sandboxes on a remote Docker host. + +Supported CI providers: + +- Bitbucket Pipelines +- CircleCI +- GitLab This image(s) is part of the [Docksal](http://docksal.io) image library. -## Docksal CI Sandboxes +## Docksal Sandboxes + +Docksal Sandboxes are continuous integration environments powered by Docksal. +They can be provisioned for any git branch and are feature-wise identical to project's local Docksal environment. -For any Docksal powered project enabling Bitbucket Pipelines and dropping [this](bitbucket-pipelines.yml) file -into the project repo will enable per-branch sandbox provisioning. -URLs to sandbox environments can be found in the build logs and can also published to a desired Slack channel. +Use cases: + +- automated testing (full stack) +- manual testing +- enhanced pull request review experience +- demos + +URLs to sandbox environments can be found in the build logs and can also be published to a Slack channel. ## Image variants and versions ### Stable -- `docksal/bitbucket-pipelines-agent` - basic (bash, curl, git) -- `docksal/bitbucket-pipelines-agent:php` - basic + php stack tools (composer, drush, drupal console, wp-cli, etc) -- `docksal/bitbucket-pipelines-agent:1.0` - basic, specific version -- `docksal/bitbucket-pipelines-agent:1.0-php` - php, specific version +- `docksal/bitbucket-pipelines-agent` - basic (bash, curl, git, etc.), latest version +- `docksal/bitbucket-pipelines-agent:php` - basic + php stack tools (composer, drush, drupal console, wp-cli, etc.), latest version +- `docksal/bitbucket-pipelines-agent:1.0` - basic, specific stable version +- `docksal/bitbucket-pipelines-agent:1.0-php` - php, specific stable version ### Development -- `docksal/bitbucket-pipelines-agent:edge` -- `docksal/bitbucket-pipelines-agent:edge-php` +- `docksal/bitbucket-pipelines-agent:edge` - base, latest development version +- `docksal/bitbucket-pipelines-agent:edge-php`, php, latest development version ## Configuration @@ -52,6 +66,15 @@ E.g. cloning/pushing a repo, running commands over SSH on a remote deployment en Other features and integrations are usually configured at the Bitbucket repo level. See below. +### Bitbucket Pipelines + +Copy the example [bitbucket-pipelines.yml](examples/bitbucket-pipelines/bitbucket-pipelines.yml) file into the project +repo and adjust as necessary. + +### CircleCI + +Copy the example [config.yml](examples/.circleci/config.yml) file into the project repo and adjust as necessary. + ## Basic HTTP Auth @@ -83,10 +106,103 @@ A public or private channel in Slack, e.g. `SLACK_CHANNEL #project-name-bots` ### Usage -`slack 'message' ['#channel'] ['webhook_url']` +```bash +slack 'message' ['#channel'] ['webhook_url'] +``` Channel and webhook url can be passed via environment variables. See above. ### Limitations Incoming Webhook integration won't work for private channels, which the owner of the integration does not belong to. + + +## Build artifact storage + +Build artifacts can be stored in an AWS S3 bucket. + +### Configuration + +Set the following environment variables at the organization or repo level: + +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` +- `ARTIFACTS_BUCKET_NAME` +- `ARTIFACTS_BASE_URL` (optional) +- `BITBUCKET_API_AUTH` (optional) + +### Usage + +To upload artifacts to the configured S3 bucket use the `build-acp` command. + +```bash +build-acp /source/path/ +``` + +There is no file browsing capability available for private S3 buckets. +An `index.html` file is used as the directory index, however it has to be created/generated manually. +When uploading a prepared artifacts folder with the `index.html` file in it, add a trailing slash to the source path to +upload the contents of the source folder vs the folder itself. + +You can upload additional folders/files by running the command multiple times. + +The optional `destination` argument can be used to define a subdirectory at the destination in the bucket. + +```bash +build-acp /source/path/ destination/path +``` + +### Advanced usage + +**Base URL** + +The base URL is derived from `ARTIFACTS_BUCKET_NAME` as follows (assuming AWS S3 `us-east-1` region by default): + +``` +https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com +``` + +It can be overridden via the optional `ARTIFACTS_BASE_URL` configuration variable at the organization/repo level: + +``` +ARTIFACTS_BASE_URL = https://artifacts.example.com +``` + +**Upload path** + +The upload path is unique for each commit and is derived as follows: + +```bash +${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH} +``` + +In certain cases you may want to store build artifacts per branch instead of per commit. +To do this, override the `ARTIFACTS_BUCKET_PATH` variable before calling the `build-acp` command: + +```bash +export ARTIFACTS_BUCKET_PATH="${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}" +build-acp my-artifacts/ +``` + +**Posting build artifact URLs to Bitbucket** + +If the `BITBUCKET_API_AUTH` is set, the URL to the artifacts will be posted back to Bitbucket via +[Bitbucket Build Status API](https://blog.bitbucket.org/2015/11/18/introducing-the-build-status-api-for-bitbucket-cloud/). + +See [this article](https://confluence.atlassian.com/bitbucket/publish-and-link-your-build-artifacts-872137736.html) for +information on how to get `BITBUCKET_API_AUTH` (steps 1 and 2 only). + +### Security + +If a bucket does not exist, it will be created automatically (with no public access). Existing bucket access permissions +are not automatically adjusted. It's up to you whether you want to keep them open or not. + +When artifacts are uploaded, the destination artifact folder in the bucket is set to be publicly accessible. +Anyone with the direct link will be able to access the artifacts, but will not be able to browse the list of all +available artifact folders in the bucket (so long as the bucket itself is set to be private). + +The URL by default includes a git commit hash, which serves as an authentication token (the URL is impossible to guess). +This provides a simple yet efficient level of security for artifacts. + +To add an additional level of security follow [this guide](https://medium.com/@lmakarov/serverless-password-protecting-a-static-website-in-an-aws-s3-bucket-bfaaa01b8666) +to set up username/password access to S3 via CloudFront and Lambda@Edge. diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml deleted file mode 100644 index d11d071..0000000 --- a/bitbucket-pipelines.yml +++ /dev/null @@ -1,15 +0,0 @@ -image: docksal/bitbucket-pipelines-agent:1.0 - -pipelines: - # Build sandboxes for all branches - default: - - step: - script: - # Initialize the agent configuration - - source build-env - # Initialize the remote sandbox environment - - build-init - - slack "Started sandbox build for branch *${BITBUCKET_BRANCH}*" - # Run fin init on the remote docker host to provision a sandbox - - ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" - - slack "Completed sandbox build for branch *${BITBUCKET_BRANCH}*:\n" diff --git a/examples/.circleci/config.yml b/examples/.circleci/config.yml new file mode 100644 index 0000000..6788ed8 --- /dev/null +++ b/examples/.circleci/config.yml @@ -0,0 +1,28 @@ +version: 2 + +jobs: + build: + working_directory: /home/agent/build + docker: + - image: docksal/bitbucket-pipelines-agent:edge + environment: + BASH_ENV: /home/agent/.bashrc # Tell CircleCI where to read the shared build environment variables from + steps: + - checkout + - run: + name: Configure local build environment + command: | + touch $BASH_ENV + echo 'source build-env' >> $BASH_ENV + - run: + name: Sandbox provisioning + command: | + build-init + ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" + +workflows: + version: 2 + default-workflow: + jobs: + - build: + context: org-global # Load org level environment variables from CircleCI diff --git a/examples/bitbucket-pipelines/bitbucket-pipelines.yml b/examples/bitbucket-pipelines/bitbucket-pipelines.yml new file mode 100644 index 0000000..6d0359e --- /dev/null +++ b/examples/bitbucket-pipelines/bitbucket-pipelines.yml @@ -0,0 +1,11 @@ +image: docksal/bitbucket-pipelines-agent:1.0 + +pipelines: + default: + - step: + script: + - source build-env # Initialize the agent configuration + - build-init # Initialize the remote sandbox environment + - slack "Started sandbox build for branch *${BITBUCKET_BRANCH}*" # post to Slack + - ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" # Run fin init on the remote docker host to provision a sandbox + - slack "Completed sandbox build for branch *${BITBUCKET_BRANCH}*:\n" From a3e2d0f3c18b12ff15f52a285b5b45e49b2731b5 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 20 Dec 2017 21:26:43 -0800 Subject: [PATCH 36/71] Renaming to docksal/ci-agent --- Makefile | 22 +++++++++---------- README.md | 12 +++++----- examples/.circleci/config.yml | 2 +- .../bitbucket-pipelines.yml | 2 +- php/Dockerfile | 2 +- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 2e2640c..5d34136 100644 --- a/Makefile +++ b/Makefile @@ -2,40 +2,40 @@ VERSION ?= base -REPO = docksal/bitbucket-pipelines-agent -NAME = bitbucket-pipelines-agent +REPO = docksal/ci-agent +NAME = ci-agent .PHONY: build test push shell run start stop logs clean release build: - fin docker build -t $(REPO):$(VERSION) $(VERSION) + docker build -t $(REPO):$(VERSION) $(VERSION) test: IMAGE=$(REPO):$(VERSION) NAME=$(NAME) tests/$(VERSION).bats push: - fin docker push $(REPO):$(VERSION) + docker push $(REPO):$(VERSION) shell: clean - fin docker run --rm --name $(NAME) -it $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) /bin/bash + docker run --rm --name $(NAME) -it $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) /bin/bash exec: - fin docker exec $(NAME) $(COMMAND) + docker exec $(NAME) $(COMMAND) run: clean - fin docker run --rm --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) + docker run --rm --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) start: clean - fin docker run -d --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) top -b + docker run -d --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) top -b stop: - fin docker stop $(NAME) + docker stop $(NAME) logs: - fin docker logs $(NAME) + docker logs $(NAME) clean: - fin docker rm -f $(NAME) || true + docker rm -f $(NAME) || true release: build make push -e VERSION=$(VERSION) diff --git a/README.md b/README.md index 940a53d..3041144 100644 --- a/README.md +++ b/README.md @@ -30,15 +30,15 @@ URLs to sandbox environments can be found in the build logs and can also be publ ### Stable -- `docksal/bitbucket-pipelines-agent` - basic (bash, curl, git, etc.), latest version -- `docksal/bitbucket-pipelines-agent:php` - basic + php stack tools (composer, drush, drupal console, wp-cli, etc.), latest version -- `docksal/bitbucket-pipelines-agent:1.0` - basic, specific stable version -- `docksal/bitbucket-pipelines-agent:1.0-php` - php, specific stable version +- `docksal/ci-agent:base` - basic (bash, curl, git, etc.), latest version +- `docksal/ci-agent:php` - basic + php stack tools (composer, drush, drupal console, wp-cli, etc.), latest version +- `docksal/ci-agent:1.0-base` - basic, specific stable version +- `docksal/ci-agent:1.0-php` - php, specific stable version ### Development -- `docksal/bitbucket-pipelines-agent:edge` - base, latest development version -- `docksal/bitbucket-pipelines-agent:edge-php`, php, latest development version +- `docksal/ci-agent:edge-base` - base, latest development version +- `docksal/ci-agent:edge-php`, php, latest development version ## Configuration diff --git a/examples/.circleci/config.yml b/examples/.circleci/config.yml index 6788ed8..d43a688 100644 --- a/examples/.circleci/config.yml +++ b/examples/.circleci/config.yml @@ -4,7 +4,7 @@ jobs: build: working_directory: /home/agent/build docker: - - image: docksal/bitbucket-pipelines-agent:edge + - image: docksal/ci-agent:edge-php environment: BASH_ENV: /home/agent/.bashrc # Tell CircleCI where to read the shared build environment variables from steps: diff --git a/examples/bitbucket-pipelines/bitbucket-pipelines.yml b/examples/bitbucket-pipelines/bitbucket-pipelines.yml index 6d0359e..a862b93 100644 --- a/examples/bitbucket-pipelines/bitbucket-pipelines.yml +++ b/examples/bitbucket-pipelines/bitbucket-pipelines.yml @@ -1,4 +1,4 @@ -image: docksal/bitbucket-pipelines-agent:1.0 +image: docksal/ci-agent:edge-php pipelines: default: diff --git a/php/Dockerfile b/php/Dockerfile index af5b394..1caa092 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -1,4 +1,4 @@ -FROM docksal/bitbucket-pipelines-agent:edge +FROM docksal/ci-agent:edge # Switch to root to install some system-wide stuff USER root From 621d593655bbdf5a6f95edfacb2c2017c337b680 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 20 Dec 2017 21:30:15 -0800 Subject: [PATCH 37/71] Push to Docker Hub with Travis --- .travis.yml | 29 +++++++++++++++++++++++++---- php/Dockerfile | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a88ff6..6ae1384 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,8 @@ services: - docker env: - matrix: - - VERSION=base - - VERSION=php + global: + - REPO=docksal/ci-agent install: - curl -fsSL get.docksal.io | sh @@ -14,7 +13,29 @@ install: - fin sysinfo script: - - make && make test + # Build base first, then php + - export VERSION=base && make && make test + - export VERSION=php && make && make test + +after_success: | + if [[ "${TRAVIS_PULL_REQUEST}" == "false" ]]; then + [[ "${TRAVIS_BRANCH}" == "develop" ]] && TAG="edge-${VERSION}" + [[ "${TRAVIS_BRANCH}" == "master" ]] && TAG="${VERSION}" + [[ "${TRAVIS_TAG}" != "" ]] && TAG="${TRAVIS_TAG:1:3}-${VERSION}" + + if [[ "$TAG" != "" ]]; then + docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" + # Push edge, stable and release tags + docker tag ${REPO}:${VERSION} ${REPO}:${TAG} + docker push ${REPO}:${TAG} + + # Push "latest" tag + if [[ "${TRAVIS_BRANCH}" == "master" ]] && [[ "${VERSION}" == "base" ]]; then + docker tag ${REPO}:${VERSION} ${REPO}:latest + docker push ${REPO}:latest + fi + fi + fi after_failure: - make logs diff --git a/php/Dockerfile b/php/Dockerfile index 1caa092..7976e6c 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -1,4 +1,4 @@ -FROM docksal/ci-agent:edge +FROM docksal/ci-agent:base # Switch to root to install some system-wide stuff USER root From 8b1e5ac56ce5f4c8a970f6949f7d9959ea976a36 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Wed, 3 Jan 2018 12:03:22 -0800 Subject: [PATCH 38/71] Post sandbox URL to Bitbucket build status API --- base/bin/build-init | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/base/bin/build-init b/base/bin/build-init index 56786d9..15ea16a 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -20,3 +20,14 @@ if [[ $HTTP_USER != "" ]] && [[ $HTTP_PASS != "" ]]; then fi #echo "Launching fin init..." #ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" + +# Post sandbox URL to Bitbucket build status API +# Note: this happens before the actual sandbox is up and does not reflect the actual sandbox build status, +# just provide an easier way to view the sandbox URL in Bitbucket UI (instead of digging through build logs). +if [[ "${BITBUCKET_REPO_SLUG}" != "" ]] && [[ "${BITBUCKET_API_AUTH}" != "" ]]; then + echo "Posting sandbox URL to Bitbucket..." + + BUILD_STATUS_PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Sandbox environment\", \"url\": \"${DOMAIN}\"}" + BITBUCKET_API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" + curl -sS -H "Content-Type: application/json" -X POST --user "${BITBUCKET_API_AUTH}" -d "${BUILD_STATUS_PAYLOAD}" "${BITBUCKET_API_URL}" >/dev/null +fi From 9dba150a8301e65819612f342d821c62c62ba807 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 5 Jan 2018 15:38:40 -0800 Subject: [PATCH 39/71] Use curly brackets --- base/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index 0ea71aa..be9215a 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -15,8 +15,8 @@ RUN \ # Install glibc libraries needed for docker-compose apk update && apk add --no-cache openssl ca-certificates && \ curl -L https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub && \ - curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ - apk add --no-cache glibc-$GLIBC.apk && rm glibc-$GLIBC.apk && \ + curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC}/glibc-${GLIBC}.apk && \ + apk add --no-cache glibc-${GLIBC}.apk && rm glibc-${GLIBC}.apk && \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib @@ -24,8 +24,8 @@ ARG DOCKER_VERSION=17.09.1-ce ARG DOCKER_COMPOSE_VERSION=1.18.0 RUN \ # Install docker cli - curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz" && \ - tar zxf docker-$DOCKER_VERSION.tgz && mv docker/docker /usr/local/bin && rm -rf docker-$DOCKER_VERSION* && \ + curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" && \ + tar zxf docker-${DOCKER_VERSION}.tgz && mv docker/docker /usr/local/bin && rm -rf docker-${DOCKER_VERSION}* && \ # Install docker-compose cli curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose && \ chmod +x /usr/local/bin/docker-compose && \ From 320167243de3405a34ab70ca4e4ec76a32e6cc17 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 5 Jan 2018 16:51:54 -0800 Subject: [PATCH 40/71] User AGENT_HOME, fix permissions during/after COPY --- base/Dockerfile | 11 ++++++----- php/Dockerfile | 14 ++++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index be9215a..d61f827 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -33,8 +33,8 @@ RUN \ curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && \ chmod +x /usr/local/bin/mc -ARG AGENT_USER=agent -ARG AGENT_HOME=/home/agent +ENV AGENT_USER=agent +ENV AGENT_HOME=/home/agent RUN \ # Create a non-root user with access to sudo adduser -h $AGENT_HOME -s /bin/bash -D $AGENT_USER && \ @@ -42,7 +42,6 @@ RUN \ # All further RUN commands will run as the "agent" user USER $AGENT_USER -ENV HOME $AGENT_HOME RUN \ mkdir -p $AGENT_HOME/.ssh && \ @@ -50,7 +49,9 @@ RUN \ COPY bin /usr/local/bin COPY config/.ssh/config $AGENT_HOME/.ssh/config -# Fix permissions after COPY -RUN sudo chown -R $AGENT_USER:$AGENT_USER $AGENT_HOME/.ssh/config +# Fix permissions after COPY (could use COPY --chown, but will still need to run chmod) +RUN \ + sudo chown $AGENT_USER:$AGENT_USER $AGENT_HOME/.ssh/config && \ + sudo chmod 600 $AGENT_HOME/.ssh/config WORKDIR $AGENT_HOME/build diff --git a/php/Dockerfile b/php/Dockerfile index 7976e6c..ddb38ab 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -37,22 +37,20 @@ RUN \ chmod +x /usr/local/bin/* # Switch back to user -ARG AGENT_USER=agent USER $AGENT_USER -ENV PATH $PATH:$HOME/.composer/vendor/bin +ENV PATH $PATH:$AGENT_HOME/.composer/vendor/bin RUN \ # Add composer bin directory to PATH - echo "\n"'PATH="$PATH:$HOME/.composer/vendor/bin"' >> $HOME/.profile && \ + echo "\n"'PATH="$PATH:$AGENT_HOME/.composer/vendor/bin"' >> $AGENT_HOME/.profile && \ # Drush modules - drush dl registry_rebuild --default-major=7 --destination=$HOME/.drush && \ + drush dl registry_rebuild --default-major=7 --destination=$AGENT_HOME/.drush && \ drush cc drush && \ # Drupal Coder w/ a matching version of PHP_CodeSniffer composer global require drupal/coder && \ - phpcs --config-set installed_paths $HOME/.composer/vendor/drupal/coder/coder_sniffer && \ + phpcs --config-set installed_paths $AGENT_HOME/.composer/vendor/drupal/coder/coder_sniffer && \ # Cleanup composer clear-cache -COPY config/.drush $HOME/.drush -# Fix permissions after COPY -RUN sudo chown -R $AGENT_USER:$AGENT_USER $HOME +# Cannot use variables with COPY --chown. See https://github.com/moby/moby/issues/35018 +COPY --chown=agent:agent config/.drush $AGENT_HOME/.drush From cb952908882a5f4f2da38c2c6cee9c3e96faae61 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Fri, 5 Jan 2018 16:52:50 -0800 Subject: [PATCH 41/71] Fix only php tag being pushed to Docker Hub --- .travis.yml | 23 +++++------------------ scripts/push-image.sh | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 18 deletions(-) create mode 100755 scripts/push-image.sh diff --git a/.travis.yml b/.travis.yml index 6ae1384..01ea10f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,27 +14,14 @@ install: script: # Build base first, then php - - export VERSION=base && make && make test - - export VERSION=php && make && make test + - make VERSION=base && make test VERSION=base + - make VERSION=php && make test VERSION=php after_success: | + # Push images to Docker Hub if [[ "${TRAVIS_PULL_REQUEST}" == "false" ]]; then - [[ "${TRAVIS_BRANCH}" == "develop" ]] && TAG="edge-${VERSION}" - [[ "${TRAVIS_BRANCH}" == "master" ]] && TAG="${VERSION}" - [[ "${TRAVIS_TAG}" != "" ]] && TAG="${TRAVIS_TAG:1:3}-${VERSION}" - - if [[ "$TAG" != "" ]]; then - docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" - # Push edge, stable and release tags - docker tag ${REPO}:${VERSION} ${REPO}:${TAG} - docker push ${REPO}:${TAG} - - # Push "latest" tag - if [[ "${TRAVIS_BRANCH}" == "master" ]] && [[ "${VERSION}" == "base" ]]; then - docker tag ${REPO}:${VERSION} ${REPO}:latest - docker push ${REPO}:latest - fi - fi + scripts/push-image.sh base + scripts/push-image.sh php fi after_failure: diff --git a/scripts/push-image.sh b/scripts/push-image.sh new file mode 100755 index 0000000..b2fca72 --- /dev/null +++ b/scripts/push-image.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Pushes an image to Docker Hub + +VERSION=$1 + +[[ "${TRAVIS_BRANCH}" == "develop" ]] && TAG="edge-${VERSION}" +[[ "${TRAVIS_BRANCH}" == "master" ]] && TAG="${VERSION}" +[[ "${TRAVIS_TAG}" != "" ]] && TAG="${TRAVIS_TAG:1:3}-${VERSION}" + +if [[ "$TAG" != "" ]]; then + docker login -u "${DOCKER_USER}" -p "${DOCKER_PASS}" + # Push edge, stable and release tags + docker tag ${REPO}:${VERSION} ${REPO}:${TAG} + docker push ${REPO}:${TAG} + + # Push "latest" tag + if [[ "${TRAVIS_BRANCH}" == "master" ]] && [[ "${VERSION}" == "base" ]]; then + docker tag ${REPO}:${VERSION} ${REPO}:latest + docker push ${REPO}:latest + fi +fi From 07b2c490b342838c7229d9831b9d999dd9b1899c Mon Sep 17 00:00:00 2001 From: Sean Dietrich Date: Wed, 10 Jan 2018 09:18:47 -0800 Subject: [PATCH 42/71] CircleCI: post back sandbox URL as a Pull Request comment (Github and Bitbucket) (#12) * Added option to post URL to Pull Request if using Circle CI * Posts to Github Pull Request * Posts to Bitbucket Pull Request * Update README.md --- README.md | 15 +++++++++++++++ base/bin/build-init | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/README.md b/README.md index 3041144..8964edd 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,21 @@ repo and adjust as necessary. Copy the example [config.yml](examples/.circleci/config.yml) file into the project repo and adjust as necessary. +#### Github + +This integration allows the agent to post a comment back to the provided Pull Request. + +`GITHUB_TOKEN` + +The token can be generated from the [user's account](https://github.com/settings/tokens). + +#### Bitbucket + +This integration allows the agent to post a comment back to the provided Pull Request. + +`BITBUCKET_TOKEN` + +The token can be generated from the user's settings. Instructions on creating an [app password](https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html). When storing the app password it is in the format: `USER:PASSWORD` ## Basic HTTP Auth diff --git a/base/bin/build-init b/base/bin/build-init index 15ea16a..05f52e3 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -31,3 +31,19 @@ if [[ "${BITBUCKET_REPO_SLUG}" != "" ]] && [[ "${BITBUCKET_API_AUTH}" != "" ]]; BITBUCKET_API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" curl -sS -H "Content-Type: application/json" -X POST --user "${BITBUCKET_API_AUTH}" -d "${BUILD_STATUS_PAYLOAD}" "${BITBUCKET_API_URL}" >/dev/null fi + +# Post sandbox URL to either Github or Bitbucket Pull Request +# Note: this happens before the actual sandbox is up and does not reflect the actual sandbox build status +if [[ "$CIRCLECI" != "" ]]; then + if [[ $CIRCLE_REPOSITORY_URL == *"github.com"* ]] && [[ "${GITHUB_TOKEN}" != "" ]] && [[ "${CI_PULL_REQUEST}" != "" ]]; then + echo "Posting sandbox URL to Github..." + URL="https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull/"; + PULL_REQUEST_ID=$(echo $CI_PULL_REQUEST | sed "s#${URL}##g") + curl -sS -H "Content-Type: application/json" -H "Accept: application/vnd.github.v3.full+json" -H "Authorization: token ${GITHUB_TOKEN}" -X POST --data "{\"body\": \"Sandbox environment: http://${DOMAIN}\"}" https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/issues/${PULL_REQUEST_ID}/comments + elif [[ $CIRCLE_REPOSITORY_URL == *"bitbucket.com"* ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${CI_PULL_REQUEST}" != "" ]]; then + echo "Posting sandbox URL to Bitbucket..." + URL="https://bitbucket.org/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull-requests/"; + PULL_REQUEST_ID=$(echo $CI_PULL_REQUEST | sed "s#${URL}##g") + curl -sS --user "${BITBUCKET_API_AUTH}" -X POST --data "content=\"Sandbox environment: http://${DOMAIN}\"" https://api.bitbucket.org/1.0/repositories/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pullrequests/${PULL_REQUEST_ID}/comments + fi +fi From 1467943bd134c1986240fd09d44bb3ccd4a971b3 Mon Sep 17 00:00:00 2001 From: Sean Dietrich Date: Wed, 10 Jan 2018 10:34:10 -0800 Subject: [PATCH 43/71] REMOTE_BUILD_BASE. Fixes #11 (#13) Allow overriding the base directory for all builds on the sandboxes server via REMOTE_BUILD_BASE. Defaults to `/home/ubuntu/builds` --- README.md | 5 +++++ base/bin/build-env | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8964edd..3f86ea1 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,11 @@ A secondary SSH key (base64 encoded as well), which can be used for deployments on the agent. E.g. cloning/pushing a repo, running commands over SSH on a remote deployment environment. +`REMOTE_BUILD_BASE` + +The directory location on the remote server where the repositories should be cloned down to and built. +Defaults to `/home/ubuntu/builds` + Other features and integrations are usually configured at the Bitbucket repo level. See below. ### Bitbucket Pipelines diff --git a/base/bin/build-env b/base/bin/build-env index a7a8181..f5651ef 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -103,7 +103,8 @@ ssh_tunnel_init () sandbox_server_env () { - export REMOTE_BUILD_DIR="/home/ubuntu/builds/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" + export REMOTE_BUILD_BASE=${REMOTE_BUILD_BASE:-/home/ubuntu/builds} + export REMOTE_BUILD_DIR="${REMOTE_BUILD_BASE}/$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" From cdccef9254347efb42395682b7c6e85863278ff3 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 10:49:00 -0800 Subject: [PATCH 44/71] Fix sandbox URL when posting to Bitbucket buils status API --- base/bin/build-init | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/bin/build-init b/base/bin/build-init index 05f52e3..ab65216 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -27,7 +27,7 @@ fi if [[ "${BITBUCKET_REPO_SLUG}" != "" ]] && [[ "${BITBUCKET_API_AUTH}" != "" ]]; then echo "Posting sandbox URL to Bitbucket..." - BUILD_STATUS_PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Sandbox environment\", \"url\": \"${DOMAIN}\"}" + BUILD_STATUS_PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Sandbox environment\", \"url\": \"http://${DOMAIN}\"}" BITBUCKET_API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" curl -sS -H "Content-Type: application/json" -X POST --user "${BITBUCKET_API_AUTH}" -d "${BUILD_STATUS_PAYLOAD}" "${BITBUCKET_API_URL}" >/dev/null fi From 2ac3d7bfaa34b5e848f2ccc4282fb0c8a7a2f9bb Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 12:16:47 -0800 Subject: [PATCH 45/71] Added BITBUCKETCI in build-env If set ("true") then we are running in Bitbucket Pipelines --- base/bin/build-env | 1 + 1 file changed, 1 insertion(+) diff --git a/base/bin/build-env b/base/bin/build-env index f5651ef..68fcd34 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -23,6 +23,7 @@ build_env () # Support for Bitbucket Pipelines if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then echo "Detected Bitbucket Pipelines build environment" + export BITBUCKETCI="true" export GIT_REPO_OWNER="$BITBUCKET_REPO_OWNER" export GIT_REPO_NAME="$BITBUCKET_REPO_SLUG" export GIT_REPO_URL="git@bitbucket.org:$GIT_REPO_OWNER/$GIT_REPO_NAME.git" From 6c5acba1845b04b8ccf67a4a1a85483e2c8ca44a Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 12:21:06 -0800 Subject: [PATCH 46/71] Added GIT_PR_NUMBER in build-env --- base/bin/build-env | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/base/bin/build-env b/base/bin/build-env index 68fcd34..4c1702e 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -29,6 +29,8 @@ build_env () export GIT_REPO_URL="git@bitbucket.org:$GIT_REPO_OWNER/$GIT_REPO_NAME.git" export GIT_BRANCH_NAME="$BITBUCKET_BRANCH" export GIT_COMMIT_HASH="$BITBUCKET_COMMIT" + # Bitbucket Pipelines does not work with PRs + #GIT_PR_NUMBER=? fi # Support for CircleCI 2.0 @@ -39,6 +41,17 @@ build_env () export GIT_REPO_URL="$CIRCLE_REPOSITORY_URL" export GIT_BRANCH_NAME="$CIRCLE_BRANCH" export GIT_COMMIT_HASH="$CIRCLE_SHA1" + + # Figure out the pull request number + # Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds + if [[ $CIRCLE_REPOSITORY_URL == *"github.com"* ]]; then + PR_URL="https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull/" + export GIT_PR_NUMBER=$(echo $CIRCLE_PULL_REQUEST | sed "s#${PR_URL}##g") + fi + if [[ $CIRCLE_REPOSITORY_URL == *"bitbucket.com"* ]]; then + PR_URL="https://bitbucket.org/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull-requests/" + export GIT_PR_NUMBER=$(echo $CIRCLE_PULL_REQUEST | sed "s#${PR_URL}##g") + fi fi # Support for GitLab 9.0+ @@ -49,6 +62,7 @@ build_env () export GIT_REPO_URL="$CI_REPOSITORY_URL" export GIT_BRANCH_NAME="$CI_COMMIT_REF_NAME" export GIT_COMMIT_HASH="$CI_COMMIT_SHA" + export GIT_PR_NUMBER="$CI_MERGE_REQUEST_ID" fi # For debug purposes these variables can be set manually. From f69776037074f09d689feb6f4fd509c4e14c3b46 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 12:24:55 -0800 Subject: [PATCH 47/71] Bitbucket and Github API call refactoring - BITBUCKET_API_AUTH replaced with BITBUCKET_TOKEN - Using globally defined variables from build-env (must be called to set variables before anything else can work) - updated docs - TODO: move these into functions --- README.md | 13 +++++----- base/bin/build-acp | 15 +++++++---- base/bin/build-init | 62 ++++++++++++++++++++++++++++++--------------- 3 files changed, 59 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 3f86ea1..81c9c0a 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,10 @@ The token can be generated from the [user's account](https://github.com/settings #### Bitbucket -This integration allows the agent to post a comment back to the provided Pull Request. +This integration allows the agent to post build information back to Bitbucket: + +- Post sandbox and artifact URLs in a comment to the provided Pull Request +- Post sandbox and artifact URLs via [Bitbucket Build Status API](https://blog.bitbucket.org/2015/11/18/introducing-the-build-status-api-for-bitbucket-cloud/). `BITBUCKET_TOKEN` @@ -149,7 +152,8 @@ Set the following environment variables at the organization or repo level: - `AWS_SECRET_ACCESS_KEY` - `ARTIFACTS_BUCKET_NAME` - `ARTIFACTS_BASE_URL` (optional) -- `BITBUCKET_API_AUTH` (optional) +- `GITHUB_TOKEN` (optional) +- `BITBUCKET_TOKEN` (optional) ### Usage @@ -206,12 +210,9 @@ build-acp my-artifacts/ **Posting build artifact URLs to Bitbucket** -If the `BITBUCKET_API_AUTH` is set, the URL to the artifacts will be posted back to Bitbucket via +If `BITBUCKET_TOKEN` is set, the URL to the artifacts will be posted back to Bitbucket via [Bitbucket Build Status API](https://blog.bitbucket.org/2015/11/18/introducing-the-build-status-api-for-bitbucket-cloud/). -See [this article](https://confluence.atlassian.com/bitbucket/publish-and-link-your-build-artifacts-872137736.html) for -information on how to get `BITBUCKET_API_AUTH` (steps 1 and 2 only). - ### Security If a bucket does not exist, it will be created automatically (with no public access). Existing bucket access permissions diff --git a/base/bin/build-acp b/base/bin/build-acp index 6dd036d..52d10a4 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -60,14 +60,19 @@ echo "Files:" mc ls ${destination} # Post artifacts to Bitbucket build status API -if [[ "${BITBUCKET_REPO_SLUG}" != "" ]] && [[ "${BITBUCKET_API_AUTH}" != "" ]]; then +if [[ "${BITBUCKETCI}" != "" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]]; then echo "Posting artifacts URL to Bitbucket..." BUILD_STATUS_URL="${ARTIFACTS_URL}" - BUILD_STATUS_PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Build artifacts\", \"url\": \"${BUILD_STATUS_URL}\"}" - - BITBUCKET_API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" - curl -sS -H "Content-Type: application/json" -X POST --user "${BITBUCKET_API_AUTH}" -d "${BUILD_STATUS_PAYLOAD}" "${BITBUCKET_API_URL}" >/dev/null + PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Build artifacts\", \"url\": \"${BUILD_STATUS_URL}\"}" + API_URL="https://api.bitbucket.org/2.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/commit/${GIT_COMMIT_HASH}/statuses/build" + + curl -sS --request POST \ + --header "Content-Type: application/json" \ + --user "${BITBUCKET_TOKEN}" \ + --data "${PAYLOAD}" \ + "${API_URL}" \ + >/dev/null else echo "This command only works in Bitbucket Pipelines" fi diff --git a/base/bin/build-init b/base/bin/build-init index ab65216..0307074 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -14,36 +14,58 @@ echo "Configuring sandbox settings..." ssh docker-host "cd $REMOTE_BUILD_DIR && echo COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME | tee -a .docksal/docksal-local.env" ssh docker-host "cd $REMOTE_BUILD_DIR && echo VIRTUAL_HOST=$DOMAIN | tee -a .docksal/docksal-local.env" # Basic HTTP Auth -if [[ $HTTP_USER != "" ]] && [[ $HTTP_PASS != "" ]]; then +if [[ "${HTTP_USER}" != "" ]] && [[ "${HTTP_PASS}" != "" ]]; then ssh docker-host "cd $REMOTE_BUILD_DIR && echo APACHE_BASIC_AUTH_USER=$HTTP_USER | tee -a .docksal/docksal-local.env" ssh docker-host "cd $REMOTE_BUILD_DIR && echo APACHE_BASIC_AUTH_PASS=$HTTP_PASS | tee -a .docksal/docksal-local.env" fi #echo "Launching fin init..." #ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" -# Post sandbox URL to Bitbucket build status API -# Note: this happens before the actual sandbox is up and does not reflect the actual sandbox build status, -# just provide an easier way to view the sandbox URL in Bitbucket UI (instead of digging through build logs). -if [[ "${BITBUCKET_REPO_SLUG}" != "" ]] && [[ "${BITBUCKET_API_AUTH}" != "" ]]; then - echo "Posting sandbox URL to Bitbucket..." +# Bitbucket Pipelines: Post sandbox URL to Bitbucket build status API (commit/branch level status) +# Note: this happens before the actual sandbox is up and does not reflect the actual sandbox build status. +if [[ "${BITBUCKETCI}" != "" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]]; then + echo "Posting sandbox URL to Bitbucket Status API..." - BUILD_STATUS_PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Sandbox environment\", \"url\": \"http://${DOMAIN}\"}" - BITBUCKET_API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" - curl -sS -H "Content-Type: application/json" -X POST --user "${BITBUCKET_API_AUTH}" -d "${BUILD_STATUS_PAYLOAD}" "${BITBUCKET_API_URL}" >/dev/null + PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Sandbox environment\", \"url\": \"http://${DOMAIN}\"}" + API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" + + curl -sS --request POST \ + --header "Content-Type: application/json" \ + --user "${BITBUCKET_TOKEN}" \ + --data "${PAYLOAD}" \ + "${API_URL}" >/dev/null fi -# Post sandbox URL to either Github or Bitbucket Pull Request +# CircleCI: Post sandbox URL to either Github or Bitbucket Pull Request # Note: this happens before the actual sandbox is up and does not reflect the actual sandbox build status if [[ "$CIRCLECI" != "" ]]; then - if [[ $CIRCLE_REPOSITORY_URL == *"github.com"* ]] && [[ "${GITHUB_TOKEN}" != "" ]] && [[ "${CI_PULL_REQUEST}" != "" ]]; then - echo "Posting sandbox URL to Github..." - URL="https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull/"; - PULL_REQUEST_ID=$(echo $CI_PULL_REQUEST | sed "s#${URL}##g") - curl -sS -H "Content-Type: application/json" -H "Accept: application/vnd.github.v3.full+json" -H "Authorization: token ${GITHUB_TOKEN}" -X POST --data "{\"body\": \"Sandbox environment: http://${DOMAIN}\"}" https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/issues/${PULL_REQUEST_ID}/comments - elif [[ $CIRCLE_REPOSITORY_URL == *"bitbucket.com"* ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${CI_PULL_REQUEST}" != "" ]]; then - echo "Posting sandbox URL to Bitbucket..." - URL="https://bitbucket.org/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull-requests/"; - PULL_REQUEST_ID=$(echo $CI_PULL_REQUEST | sed "s#${URL}##g") - curl -sS --user "${BITBUCKET_API_AUTH}" -X POST --data "content=\"Sandbox environment: http://${DOMAIN}\"" https://api.bitbucket.org/1.0/repositories/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pullrequests/${PULL_REQUEST_ID}/comments + # Github + if [[ "${CIRCLE_REPOSITORY_URL}" == *"github.com"* ]] && [[ "${GITHUB_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then + echo "Posting sandbox URL to Github pull request..." + + PAYLOAD="{\"body\": \"Sandbox environment: http://${DOMAIN}\"}" + API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/issues/${GIT_PR_NUMBER}/comments" + + curl -sS --request POST \ + --header "Content-Type: application/json" \ + --header "Accept: application/vnd.github.v3.full+json" \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --data "${PAYLOAD}" \ + "${API_URL}" >/dev/null + fi + + # Bitbucket + if [[ "${CIRCLE_REPOSITORY_URL}" == *"bitbucket.com"* ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then + echo "Posting sandbox URL to Bitbucket pull request..." + + PAYLOAD="content=\"Sandbox environment: http://${DOMAIN}\"" + # Posting comments to PRs is only available in BB API 1.0 + API_URL="https://api.bitbucket.org/1.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/pullrequests/${GIT_PR_NUMBER}/comments" + + curl -sS --request POST \ + --header "Content-Type: application/json" \ + --user "${BITBUCKET_TOKEN}" \ + --data "${PAYLOAD}" \ + "${API_URL}" >/dev/null fi fi From 443e140f2da694df35bc20e09f97d81559bef0c0 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 14:36:50 -0800 Subject: [PATCH 48/71] CircleCI: simplified PR number extraction --- base/bin/build-env | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 4c1702e..0bef92a 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -45,12 +45,10 @@ build_env () # Figure out the pull request number # Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds if [[ $CIRCLE_REPOSITORY_URL == *"github.com"* ]]; then - PR_URL="https://github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull/" - export GIT_PR_NUMBER=$(echo $CIRCLE_PULL_REQUEST | sed "s#${PR_URL}##g") + export GIT_PR_NUMBER="$(echo $CIRCLE_PULL_REQUEST | sed 's/.*\/pull\///')" fi - if [[ $CIRCLE_REPOSITORY_URL == *"bitbucket.com"* ]]; then - PR_URL="https://bitbucket.org/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pull-requests/" - export GIT_PR_NUMBER=$(echo $CIRCLE_PULL_REQUEST | sed "s#${PR_URL}##g") + if [[ $CIRCLE_REPOSITORY_URL == *"bitbucket.org"* ]]; then + export GIT_PR_NUMBER="$(echo $CIRCLE_PULL_REQUEST | sed 's/.*\/pull-requests\///')" fi fi From ecca4a5fb59887c6d163afeebf1eb08f36ce901f Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 14:39:11 -0800 Subject: [PATCH 49/71] CircleCI: added GIT_REPO_SERVICE Ether "github" or "bitbucket" --- base/bin/build-env | 8 ++++++-- base/bin/build-init | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 0bef92a..b5e7f32 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -42,12 +42,16 @@ build_env () export GIT_BRANCH_NAME="$CIRCLE_BRANCH" export GIT_COMMIT_HASH="$CIRCLE_SHA1" - # Figure out the pull request number - # Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds if [[ $CIRCLE_REPOSITORY_URL == *"github.com"* ]]; then + export GIT_REPO_SERVICE="github" + # Figure out the pull request number + # Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds export GIT_PR_NUMBER="$(echo $CIRCLE_PULL_REQUEST | sed 's/.*\/pull\///')" fi if [[ $CIRCLE_REPOSITORY_URL == *"bitbucket.org"* ]]; then + export GIT_REPO_SERVICE="bitbucket" + # Figure out the pull request number + # Cannot use $CIRCLE_PR_NUMBER as it's only available in forked PR builds export GIT_PR_NUMBER="$(echo $CIRCLE_PULL_REQUEST | sed 's/.*\/pull-requests\///')" fi fi diff --git a/base/bin/build-init b/base/bin/build-init index 0307074..c8fe2c7 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -40,7 +40,7 @@ fi # Note: this happens before the actual sandbox is up and does not reflect the actual sandbox build status if [[ "$CIRCLECI" != "" ]]; then # Github - if [[ "${CIRCLE_REPOSITORY_URL}" == *"github.com"* ]] && [[ "${GITHUB_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then + if [[ "${GIT_REPO_SERVICE}" == "github" ]] && [[ "${GITHUB_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then echo "Posting sandbox URL to Github pull request..." PAYLOAD="{\"body\": \"Sandbox environment: http://${DOMAIN}\"}" @@ -55,7 +55,7 @@ if [[ "$CIRCLECI" != "" ]]; then fi # Bitbucket - if [[ "${CIRCLE_REPOSITORY_URL}" == *"bitbucket.com"* ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then + if [[ "${GIT_REPO_SERVICE}" == "bitbucket" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then echo "Posting sandbox URL to Bitbucket pull request..." PAYLOAD="content=\"Sandbox environment: http://${DOMAIN}\"" From 7824cc20b9f98c0996721ef4c600c2c95482e8e5 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 15:11:53 -0800 Subject: [PATCH 50/71] CircleCI: fixed Bitbucket pull request comment posting --- base/bin/build-init | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/base/bin/build-init b/base/bin/build-init index c8fe2c7..9657eec 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -58,14 +58,12 @@ if [[ "$CIRCLECI" != "" ]]; then if [[ "${GIT_REPO_SERVICE}" == "bitbucket" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then echo "Posting sandbox URL to Bitbucket pull request..." - PAYLOAD="content=\"Sandbox environment: http://${DOMAIN}\"" + PAYLOAD="content=Sandbox environment: http://${DOMAIN}" # Posting comments to PRs is only available in BB API 1.0 API_URL="https://api.bitbucket.org/1.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/pullrequests/${GIT_PR_NUMBER}/comments" - curl -sS --request POST \ - --header "Content-Type: application/json" \ + curl -sS --request POST "${API_URL}" \ --user "${BITBUCKET_TOKEN}" \ - --data "${PAYLOAD}" \ - "${API_URL}" >/dev/null + --data "${PAYLOAD}" fi fi From 69aea3ab547006ec971cb28aed210f7669467a3b Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 15:12:24 -0800 Subject: [PATCH 51/71] Un-silenced curl calls --- base/bin/build-acp | 6 ++---- base/bin/build-init | 5 ++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/base/bin/build-acp b/base/bin/build-acp index 52d10a4..42f6e1a 100755 --- a/base/bin/build-acp +++ b/base/bin/build-acp @@ -67,12 +67,10 @@ if [[ "${BITBUCKETCI}" != "" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]]; then PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Build artifacts\", \"url\": \"${BUILD_STATUS_URL}\"}" API_URL="https://api.bitbucket.org/2.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/commit/${GIT_COMMIT_HASH}/statuses/build" - curl -sS --request POST \ + curl -sS --request POST "${API_URL}" \ --header "Content-Type: application/json" \ --user "${BITBUCKET_TOKEN}" \ - --data "${PAYLOAD}" \ - "${API_URL}" \ - >/dev/null + --data "${PAYLOAD}" else echo "This command only works in Bitbucket Pipelines" fi diff --git a/base/bin/build-init b/base/bin/build-init index 9657eec..1f87579 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -46,12 +46,11 @@ if [[ "$CIRCLECI" != "" ]]; then PAYLOAD="{\"body\": \"Sandbox environment: http://${DOMAIN}\"}" API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/issues/${GIT_PR_NUMBER}/comments" - curl -sS --request POST \ + curl -sS --request POST "${API_URL}" \ --header "Content-Type: application/json" \ --header "Accept: application/vnd.github.v3.full+json" \ --header "Authorization: token ${GITHUB_TOKEN}" \ - --data "${PAYLOAD}" \ - "${API_URL}" >/dev/null + --data "${PAYLOAD}" fi # Bitbucket From 674098c96edc819239015841ded643986f8bf2da Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 15:22:30 -0800 Subject: [PATCH 52/71] Move Github/Bitbucket notifications into build-notify --- base/bin/build-init | 46 -------------------------------------- base/bin/build-notify | 52 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 46 deletions(-) create mode 100755 base/bin/build-notify diff --git a/base/bin/build-init b/base/bin/build-init index 1f87579..6642fbd 100755 --- a/base/bin/build-init +++ b/base/bin/build-init @@ -20,49 +20,3 @@ if [[ "${HTTP_USER}" != "" ]] && [[ "${HTTP_PASS}" != "" ]]; then fi #echo "Launching fin init..." #ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" - -# Bitbucket Pipelines: Post sandbox URL to Bitbucket build status API (commit/branch level status) -# Note: this happens before the actual sandbox is up and does not reflect the actual sandbox build status. -if [[ "${BITBUCKETCI}" != "" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]]; then - echo "Posting sandbox URL to Bitbucket Status API..." - - PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Sandbox environment\", \"url\": \"http://${DOMAIN}\"}" - API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" - - curl -sS --request POST \ - --header "Content-Type: application/json" \ - --user "${BITBUCKET_TOKEN}" \ - --data "${PAYLOAD}" \ - "${API_URL}" >/dev/null -fi - -# CircleCI: Post sandbox URL to either Github or Bitbucket Pull Request -# Note: this happens before the actual sandbox is up and does not reflect the actual sandbox build status -if [[ "$CIRCLECI" != "" ]]; then - # Github - if [[ "${GIT_REPO_SERVICE}" == "github" ]] && [[ "${GITHUB_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then - echo "Posting sandbox URL to Github pull request..." - - PAYLOAD="{\"body\": \"Sandbox environment: http://${DOMAIN}\"}" - API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/issues/${GIT_PR_NUMBER}/comments" - - curl -sS --request POST "${API_URL}" \ - --header "Content-Type: application/json" \ - --header "Accept: application/vnd.github.v3.full+json" \ - --header "Authorization: token ${GITHUB_TOKEN}" \ - --data "${PAYLOAD}" - fi - - # Bitbucket - if [[ "${GIT_REPO_SERVICE}" == "bitbucket" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then - echo "Posting sandbox URL to Bitbucket pull request..." - - PAYLOAD="content=Sandbox environment: http://${DOMAIN}" - # Posting comments to PRs is only available in BB API 1.0 - API_URL="https://api.bitbucket.org/1.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/pullrequests/${GIT_PR_NUMBER}/comments" - - curl -sS --request POST "${API_URL}" \ - --user "${BITBUCKET_TOKEN}" \ - --data "${PAYLOAD}" - fi -fi diff --git a/base/bin/build-notify b/base/bin/build-notify new file mode 100755 index 0000000..25bbd02 --- /dev/null +++ b/base/bin/build-notify @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Posts sandbox environment URL to: +# - Github pull request comment +# - Bitbucket pull request comment +# - Bitbucket Status API +# +# Call after sandbox provisioning commands in your build script. + +# Bitbucket Pipelines: Post sandbox URL to Bitbucket build status API (commit/branch level status) +if [[ "${BITBUCKETCI}" != "" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]]; then + echo "Posting sandbox URL to Bitbucket Status API..." + + PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Sandbox environment\", \"url\": \"http://${DOMAIN}\"}" + API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" + + curl -sS --request POST \ + --header "Content-Type: application/json" \ + --user "${BITBUCKET_TOKEN}" \ + --data "${PAYLOAD}" \ + "${API_URL}" >/dev/null +fi + +# CircleCI: Post sandbox URL to either Github or Bitbucket Pull Request +if [[ "$CIRCLECI" != "" ]]; then + # Github + if [[ "${GIT_REPO_SERVICE}" == "github" ]] && [[ "${GITHUB_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then + echo "Posting sandbox URL to Github pull request..." + + PAYLOAD="{\"body\": \"Sandbox environment: http://${DOMAIN}\"}" + API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/issues/${GIT_PR_NUMBER}/comments" + + curl -sS --request POST "${API_URL}" \ + --header "Content-Type: application/json" \ + --header "Accept: application/vnd.github.v3.full+json" \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --data "${PAYLOAD}" + fi + + # Bitbucket + if [[ "${GIT_REPO_SERVICE}" == "bitbucket" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then + echo "Posting sandbox URL to Bitbucket pull request..." + + PAYLOAD="content=Sandbox environment: http://${DOMAIN}" + # Posting comments to PRs is only available in BB API 1.0 + API_URL="https://api.bitbucket.org/1.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/pullrequests/${GIT_PR_NUMBER}/comments" + + curl -sS --request POST "${API_URL}" \ + --user "${BITBUCKET_TOKEN}" \ + --data "${PAYLOAD}" + fi +fi From fcf16561ab48e8aa58e10649dba17c7444054bb3 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 19:29:24 -0800 Subject: [PATCH 53/71] Build status notifications build-notify can post updates via builds status API (Github and Bitbucket) and/or pull request comments Usage: build-notify --- base/bin/build-notify | 203 ++++++++++++++++++++++++++++++++---------- 1 file changed, 157 insertions(+), 46 deletions(-) diff --git a/base/bin/build-notify b/base/bin/build-notify index 25bbd02..20b3fb7 100755 --- a/base/bin/build-notify +++ b/base/bin/build-notify @@ -1,52 +1,163 @@ #!/usr/bin/env bash -# Posts sandbox environment URL to: -# - Github pull request comment -# - Bitbucket pull request comment -# - Bitbucket Status API -# -# Call after sandbox provisioning commands in your build script. - -# Bitbucket Pipelines: Post sandbox URL to Bitbucket build status API (commit/branch level status) -if [[ "${BITBUCKETCI}" != "" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]]; then - echo "Posting sandbox URL to Bitbucket Status API..." - - PAYLOAD="{\"key\": \"artifacts\", \"state\": \"SUCCESSFUL\", \"name\": \"Sandbox environment\", \"url\": \"http://${DOMAIN}\"}" - API_URL="https://api.bitbucket.org/2.0/repositories/${BITBUCKET_REPO_OWNER}/${BITBUCKET_REPO_SLUG}/commit/${BITBUCKET_COMMIT}/statuses/build" - - curl -sS --request POST \ +# Build status notifications +# Can post updates via builds status API (Github and Bitbucket) and/or pull request comments + +#-------------------------- SETTINGS ---------------------------- + +export CONTEXT="ci/docksal" # Context for notifications +export URL="http://${DOMAIN}" # Sandbox URL +export STATUS_API=${STATUS_API:-1} # Status API notifications enabled by default +export PR_COMMENT=${PR_COMMENT:-0} # PR comment notifications disabled by default + +#-------------------------- FUNCTIONS ---------------------------- + +# Calls Github status API (commit/branch level status updates) +# API docs: https://developer.github.com/v3/repos/statuses/ +# +# @param $1 context - unique key for this status (e.g. sandbox, artifacts) +# @param $2 state - error, failure, pending, or success +# @param $3 description - description of the status update +# @param $4 url - status link URL +# +# Example: github_status_api "sandbox" "pending" "Started building sandbox environment" "http://${DOMAIN}" +# +github_status_api () +{ + [[ "${GIT_REPO_SERVICE}" != "github" ]] && return + [[ "${GITHUB_TOKEN}" == "" ]] && echo "Warning: GITHUB_TOKEN not set" && return + + local context="$1" + local state="$2" + local description="$3" + local url="$4" + + echo "Notifying Github Status API: ${key} => ${state}" + + PAYLOAD="{\"state\": \"${state}\",\"target_url\": \"${url}\",\"description\": \"${description}\",\"context\": \"${context}\"}" + API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/statuses/${GIT_COMMIT_HASH}" + + curl -sS --request POST "${API_URL}" --header "Authorization: token ${GITHUB_TOKEN}" --data "${PAYLOAD}" +} + +# Calls Bitbucket Pipelines status API (commit/branch level status updates) +# API docs: https://developer.atlassian.com/bitbucket/api/2/reference/resource/repositories/%7Busername%7D/%7Brepo_slug%7D/commit/%7Bnode%7D/statuses/build +# +# @param $1 context - unique key for this status (e.g. sandbox, artifacts) +# @param $2 state - SUCCESSFUL, FAILED, INPROGRESS, STOPPED +# @param $3 description - description of the status update +# @param $4 url - status link URL +# +# Example: bitbucket_status_api "sandbox" "INPROGRESS" "Sandbox environment" "http://${DOMAIN}" +# +bitbucket_status_api () +{ + [[ "${GIT_REPO_SERVICE}" != "bitbucket" ]] && return + [[ "${BITBUCKET_TOKEN}" == "" ]] && echo "Warning: BITBUCKET_TOKEN not set" && return + + local context="$1" + local state="$2" + local description="$3" + local url="$4" + + echo "Notifying Bitbucket Status API: ${key} => ${state}" + + PAYLOAD="{\"key\": \"${context}\", \"state\": \"${state}\", \"description\": \"${description}\", \"url\": \"${url}\"}" + API_URL="https://api.bitbucket.org/2.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/commit/${GIT_COMMIT_HASH}/statuses/build" + + curl -sS --request POST "${API_URL}" \ --header "Content-Type: application/json" \ --user "${BITBUCKET_TOKEN}" \ - --data "${PAYLOAD}" \ - "${API_URL}" >/dev/null -fi - -# CircleCI: Post sandbox URL to either Github or Bitbucket Pull Request -if [[ "$CIRCLECI" != "" ]]; then - # Github - if [[ "${GIT_REPO_SERVICE}" == "github" ]] && [[ "${GITHUB_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then - echo "Posting sandbox URL to Github pull request..." - - PAYLOAD="{\"body\": \"Sandbox environment: http://${DOMAIN}\"}" - API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/issues/${GIT_PR_NUMBER}/comments" - - curl -sS --request POST "${API_URL}" \ - --header "Content-Type: application/json" \ - --header "Accept: application/vnd.github.v3.full+json" \ - --header "Authorization: token ${GITHUB_TOKEN}" \ - --data "${PAYLOAD}" - fi + --data "${PAYLOAD}" +} + +# Posts a comment to a pull request in Github +# @param $1 - comment +github_pr_comment () +{ + [[ "${GIT_REPO_SERVICE}" != "github" ]] && return + [[ "${GIT_PR_NUMBER}" == "" ]] && return + [[ "${GITHUB_TOKEN}" == "" ]] && echo "Warning: GITHUB_TOKEN not set" && return + + local comment="$1" - # Bitbucket - if [[ "${GIT_REPO_SERVICE}" == "bitbucket" ]] && [[ "${BITBUCKET_TOKEN}" != "" ]] && [[ "${GIT_PR_NUMBER}" != "" ]]; then - echo "Posting sandbox URL to Bitbucket pull request..." - - PAYLOAD="content=Sandbox environment: http://${DOMAIN}" - # Posting comments to PRs is only available in BB API 1.0 - API_URL="https://api.bitbucket.org/1.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/pullrequests/${GIT_PR_NUMBER}/comments" - - curl -sS --request POST "${API_URL}" \ - --user "${BITBUCKET_TOKEN}" \ - --data "${PAYLOAD}" + echo "Posting sandbox URL to Github pull request..." + PAYLOAD="{\"body\": \"${comment}\"}" + API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/issues/${GIT_PR_NUMBER}/comments" + + curl -sS --request POST "${API_URL}" \ + --header "Content-Type: application/json" \ + --header "Accept: application/vnd.github.v3.full+json" \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --data "${PAYLOAD}" +} + +# Posts a comment to a pull request in Bitbucket +# @param $1 - comment +bitbucket_pr_comment () +{ + [[ "${GIT_REPO_SERVICE}" != "bitbucket" ]] && return + [[ "${GIT_PR_NUMBER}" == "" ]] && return + [[ "${BITBUCKET_TOKEN}" == "" ]] && echo "Warning: BITBUCKET_TOKEN not set" && return + + local comment="$1" + + echo "Posting sandbox URL to Bitbucket pull request..." + PAYLOAD="content=${comment}" + # Posting comments to PRs is only available in BB API 1.0 + API_URL="https://api.bitbucket.org/1.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/pullrequests/${GIT_PR_NUMBER}/comments" + + curl -sS --request POST "${API_URL}" \ + --user "${BITBUCKET_TOKEN}" \ + --data "${PAYLOAD}" +} + +notify_pending () +{ + description="Started building sandbox environment" + + [[ "$GIT_REPO_SERVICE" == "bitbucket" ]] && bitbucket_status_api "$CONTEXT" "INPROGRESS" "$description" "$URL" + [[ "$GIT_REPO_SERVICE" == "github" ]] && github_status_api "$CONTEXT" "pending" "$description" "$URL" +} + +notify_success () +{ + description="Sandbox environment build completed" + + [[ "$GIT_REPO_SERVICE" == "bitbucket" ]] && bitbucket_status_api "$CONTEXT" "SUCCESSFUL" "$description" "$URL" + [[ "$GIT_REPO_SERVICE" == "github" ]] && github_status_api "$CONTEXT" "success" "$description" "$URL" + + # Post comment to a pull request on success only + # Note: pull request builds are not supported by Bitbucket Pipelines + if [[ "$CIRCLECI" != "" ]] && [[ "$PR_COMMENT" != 0 ]]; then + comment="$description: http://${DOMAIN}" + [[ "${GIT_REPO_SERVICE}" == "github" ]] && github_pr_comment "$comment" + [[ "${GIT_REPO_SERVICE}" == "bitbucket" ]] && bitbucket_pr_comment "$comment" fi -fi +} + +notify_failure () +{ + description="Sandbox environment build failed" + + [[ "$GIT_REPO_SERVICE" == "bitbucket" ]] && bitbucket_status_api "$CONTEXT" "FAILED" "$description" "$URL" + [[ "$GIT_REPO_SERVICE" == "github" ]] && github_status_api "$CONTEXT" "failure" "$description" "$URL" +} + +#-------------------------- RUNTIME STARTS HERE ---------------------------- + +# Post build state via status API +case "$1" in + pending) + notify_pending + ;; + success) + notify_success + ;; + failure) + notify_failure + ;; + *) + echo "Usage: build-notify " + ;; +esac From f28ecabb36a88106bd99d53bdcd8e92f767636f0 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 19:55:18 -0800 Subject: [PATCH 54/71] build-env: added GIT_REPO_SERVICE for bitbucket and gitlab --- base/bin/build-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/bin/build-env b/base/bin/build-env index b5e7f32..313aca3 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -24,6 +24,7 @@ build_env () if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then echo "Detected Bitbucket Pipelines build environment" export BITBUCKETCI="true" + export GIT_REPO_SERVICE="bitbucket" export GIT_REPO_OWNER="$BITBUCKET_REPO_OWNER" export GIT_REPO_NAME="$BITBUCKET_REPO_SLUG" export GIT_REPO_URL="git@bitbucket.org:$GIT_REPO_OWNER/$GIT_REPO_NAME.git" @@ -59,6 +60,7 @@ build_env () # Support for GitLab 9.0+ if [[ "$GITLAB_CI" != "" ]]; then echo "Detected GitLabCI build environment" + export GIT_REPO_SERVICE="gitlab" export GIT_REPO_OWNER="$CI_PROJECT_NAMESPACE" export GIT_REPO_NAME="$CI_PROJECT_NAME" export GIT_REPO_URL="$CI_REPOSITORY_URL" From 235b87e15bd65158cd9bac8873097c455e812f13 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 20:04:41 -0800 Subject: [PATCH 55/71] Fixes --- base/bin/build-notify | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/base/bin/build-notify b/base/bin/build-notify index 20b3fb7..243e5b2 100755 --- a/base/bin/build-notify +++ b/base/bin/build-notify @@ -32,12 +32,16 @@ github_status_api () local description="$3" local url="$4" - echo "Notifying Github Status API: ${key} => ${state}" + echo "Notifying Github Status API: ${context} => ${state}" PAYLOAD="{\"state\": \"${state}\",\"target_url\": \"${url}\",\"description\": \"${description}\",\"context\": \"${context}\"}" API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/statuses/${GIT_COMMIT_HASH}" - curl -sS --request POST "${API_URL}" --header "Authorization: token ${GITHUB_TOKEN}" --data "${PAYLOAD}" + curl -sS --request POST "${API_URL}" \ + --header "Content-Type: application/json" \ + --header "Accept: application/vnd.github.v3.full+json" \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --data "${PAYLOAD}" } # Calls Bitbucket Pipelines status API (commit/branch level status updates) @@ -60,7 +64,7 @@ bitbucket_status_api () local description="$3" local url="$4" - echo "Notifying Bitbucket Status API: ${key} => ${state}" + echo "Notifying Bitbucket Status API: ${context} => ${state}" PAYLOAD="{\"key\": \"${context}\", \"state\": \"${state}\", \"description\": \"${description}\", \"url\": \"${url}\"}" API_URL="https://api.bitbucket.org/2.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/commit/${GIT_COMMIT_HASH}/statuses/build" From 0a4d5add97337a6fe20c0e3edef8562d33014dd2 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 21:59:58 -0800 Subject: [PATCH 56/71] build-notify: return 0 Necessary when doing multiple if statements in a row, since the last one sets the exit code --- base/bin/build-notify | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/base/bin/build-notify b/base/bin/build-notify index 243e5b2..cb5e58a 100755 --- a/base/bin/build-notify +++ b/base/bin/build-notify @@ -122,6 +122,8 @@ notify_pending () [[ "$GIT_REPO_SERVICE" == "bitbucket" ]] && bitbucket_status_api "$CONTEXT" "INPROGRESS" "$description" "$URL" [[ "$GIT_REPO_SERVICE" == "github" ]] && github_status_api "$CONTEXT" "pending" "$description" "$URL" + + return 0 } notify_success () @@ -138,6 +140,8 @@ notify_success () [[ "${GIT_REPO_SERVICE}" == "github" ]] && github_pr_comment "$comment" [[ "${GIT_REPO_SERVICE}" == "bitbucket" ]] && bitbucket_pr_comment "$comment" fi + + return 0 } notify_failure () @@ -146,6 +150,8 @@ notify_failure () [[ "$GIT_REPO_SERVICE" == "bitbucket" ]] && bitbucket_status_api "$CONTEXT" "FAILED" "$description" "$URL" [[ "$GIT_REPO_SERVICE" == "github" ]] && github_status_api "$CONTEXT" "failure" "$description" "$URL" + + return 0 } #-------------------------- RUNTIME STARTS HERE ---------------------------- From ab001ee83a79e581b34c1f2f3cb33f7597edd1a2 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 11 Jan 2018 23:09:51 -0800 Subject: [PATCH 57/71] Updated docs [ci skip] --- README.md | 67 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 81c9c0a..e4cf723 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,10 @@ URLs to sandbox environments can be found in the build logs and can also be publ - `docksal/ci-agent:edge-php`, php, latest development version -## Configuration +## Global Configuration -The following required variables should be configured at the Bitbucket organization level (this way all -project repos will have access to them). +The following required variables are usually configured at the organization level. This way, all project repos will +have access to them. They can as well be configured at the repo level. `DOCKSAL_HOST` or `DOCKSAL_HOST_IP` @@ -69,49 +69,64 @@ E.g. cloning/pushing a repo, running commands over SSH on a remote deployment en The directory location on the remote server where the repositories should be cloned down to and built. Defaults to `/home/ubuntu/builds` -Other features and integrations are usually configured at the Bitbucket repo level. See below. +`GITHUB_TOKEN` and `BITBUCKET_TOKEN` -### Bitbucket Pipelines +Used for access to post sandbox URLs via build status API as well as comments on pull requests. -Copy the example [bitbucket-pipelines.yml](examples/bitbucket-pipelines/bitbucket-pipelines.yml) file into the project -repo and adjust as necessary. +For Github, the token can be generated from the [user's account](https://github.com/settings/tokens). +Set access to "repo" (http://take.ms/nMqcW). -### CircleCI +For Bitbucket, the token can be generated from the user's settings. Instructions on creating an [app password](https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html). +Set access to "Repositories: Write", "Pull requests: Write" (http://take.ms/98BG5). +When storing the app password it is in the format: `USER:PASSWORD`. -Copy the example [config.yml](examples/.circleci/config.yml) file into the project repo and adjust as necessary. +Other features and integrations are usually configured at the repo level. See below. -#### Github -This integration allows the agent to post a comment back to the provided Pull Request. +## Project configuration -`GITHUB_TOKEN` +For Bitbucket Pipelines, copy the example [bitbucket-pipelines.yml](examples/bitbucket-pipelines/bitbucket-pipelines.yml) +file into the project repo and adjust as necessary. -The token can be generated from the [user's account](https://github.com/settings/tokens). +For CircleCI, copy the example [config.yml](examples/.circleci/config.yml) file into the project repo and adjust as necessary. -#### Bitbucket +## Feature: Basic HTTP Auth -This integration allows the agent to post build information back to Bitbucket: +Protect sandboxes from public access using Basic HTTP authentication. -- Post sandbox and artifact URLs in a comment to the provided Pull Request -- Post sandbox and artifact URLs via [Bitbucket Build Status API](https://blog.bitbucket.org/2015/11/18/introducing-the-build-status-api-for-bitbucket-cloud/). +### Configuration -`BITBUCKET_TOKEN` +Set the following environment variables at the repo level: -The token can be generated from the user's settings. Instructions on creating an [app password](https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html). When storing the app password it is in the format: `USER:PASSWORD` +- `HTTP_USER` +- `HTTP_PASS` -## Basic HTTP Auth -Protect sandboxes from public access using Basic HTTP authentication. +## Feature: Build status notifications + +This integration allows the agent to post build status updates and sandbox URL via Github/Bitbucket build status API. +For CircleCI, it is also possible to enable posting the sandbox URL as a comment in pull requests. ### Configuration -Set the following environment variables at the repo level: +`GITHUB_TOKEN` or `BITBUCKET_TOKEN` must be configured respectively (either globally or at the repo level). -- `HTTP_USER` -- `HTTP_PASS` +### Usage + +`build-notify ` + +Place the triggers right before and right after `fin init` call in your build script, e.g. + +```bash +build-notify pending +ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" +if [[ $? == 0 ]]; then build-notify success; else build-notify failure; fi +``` + +To enable posting sandbox URLs in comments on pull requests, do `export PR_COMMENT=1` prior to calling `build-notify` -## Slack integration +## Feature: Slack notifications This integrations allows the agent to post messages to a given Slack channel. It can be used for notification purposes when a build is started, completed, failed, etc. @@ -140,7 +155,7 @@ Channel and webhook url can be passed via environment variables. See above. Incoming Webhook integration won't work for private channels, which the owner of the integration does not belong to. -## Build artifact storage +## Feature: Build artifact storage Build artifacts can be stored in an AWS S3 bucket. From b26d77a93af55fc98c29a6af8488c9b846de97fe Mon Sep 17 00:00:00 2001 From: Sean Dietrich Date: Mon, 15 Jan 2018 09:20:20 -0800 Subject: [PATCH 58/71] Update README.md Added undocumented variables to README.md: DOCKSAL_HOST_USER, GIT_USER_EMAIL, GIT_USER_NAME --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e4cf723..52ea074 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,10 @@ The address of the remote Docksal host, which will be hosting sandboxes. Configu If using `DOCKSAL_HOST`, make sure the domain is configured as a wildcard DNS entry. If using `DOCKSAL_HOST_IP`, the agent will use `xip.io` for dynamic wildcard domain names for sandboxes. +`DOCKSAL_HOST_USER` + +The user's name that should have access to the remote Docksal host. Defaults to `ubuntu`. + `DOCKSAL_HOST_SSH_KEY` A base64 encoded private SSH key used to access the remote Docksal host. @@ -61,7 +65,7 @@ tutorial for details. `CI_SSH_KEY` A secondary SSH key (base64 encoded as well), which can be used for deployments and other remote operations run directly -on the agent. +on the agent. E.g. cloning/pushing a repo, running commands over SSH on a remote deployment environment. `REMOTE_BUILD_BASE` @@ -80,6 +84,14 @@ For Bitbucket, the token can be generated from the user's settings. Instructions Set access to "Repositories: Write", "Pull requests: Write" (http://take.ms/98BG5). When storing the app password it is in the format: `USER:PASSWORD`. +`GIT_USER_EMAIL` + +The user's email to perform Git operations as. Defaults to `ci@docksal.io` + +`GIT_USER_NAME` + +The user's name to perform Git operations as. Defaults to `Docksal CI` + Other features and integrations are usually configured at the repo level. See below. From 69fa14ce96ae5586d5d279ed0fe5851f1868453b Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 15 Jan 2018 11:07:25 -0800 Subject: [PATCH 59/71] Fix build-notify failure case in docs [ci skip] --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e4cf723..52b3c26 100644 --- a/README.md +++ b/README.md @@ -119,8 +119,8 @@ Place the triggers right before and right after `fin init` call in your build sc ```bash build-notify pending -ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" -if [[ $? == 0 ]]; then build-notify success; else build-notify failure; fi +ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" || ( build-notify failure && exit 1 ) +build-notify success ``` To enable posting sandbox URLs in comments on pull requests, do `export PR_COMMENT=1` prior to calling `build-notify` From 03336eb5fafd16e51f294cac0df82fa98ccd8128 Mon Sep 17 00:00:00 2001 From: Sean Dietrich Date: Mon, 15 Jan 2018 23:00:32 -0800 Subject: [PATCH 60/71] Update slack changed slack configuration to make it more customizable. --- base/bin/slack | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/base/bin/slack b/base/bin/slack index ac3fb84..9ca379f 100755 --- a/base/bin/slack +++ b/base/bin/slack @@ -3,6 +3,10 @@ MESSAGE="${1}" SLACK_CHANNEL="${SLACK_CHANNEL:-$2}" SLACK_WEBHOOK_URL="${SLACK_WEBHOOK_URL:-$3}" +SLACK_USER="${SLACK_USER:-$4}" +SLACK_USER="${SLACK_USER:-Docksal CI}" +SLACK_ICON="${SLACK_ICON:-$5}" +SLACK_ICON="${SLACK_ICON:-:desktop_computer:}" if [[ "$MESSAGE" == "" ]] || [[ "$SLACK_CHANNEL" == "" ]] || [[ "$SLACK_WEBHOOK_URL" == "" ]]; then echo "Usage: slack 'message' ['#channel'] ['webhook_url']" @@ -12,4 +16,4 @@ if [[ "$MESSAGE" == "" ]] || [[ "$SLACK_CHANNEL" == "" ]] || [[ "$SLACK_WEBHOOK_ exit 1 fi -curl -X POST --data-urlencode "payload={'channel': '${SLACK_CHANNEL}', 'username': 'Bitbucket Pipelines', 'text': '${MESSAGE}', 'icon_emoji': ':bitbucket:'}" "${SLACK_WEBHOOK_URL}" +curl -X POST --data-urlencode "payload={'channel': '${SLACK_CHANNEL}', 'username': '${SLACK_USER}', 'text': '${MESSAGE}', 'icon_emoji': '${SLACK_ICON}'}" "${SLACK_WEBHOOK_URL}" From 377d3886ad7c139c1473f881f3de25da3ca08807 Mon Sep 17 00:00:00 2001 From: Sean Dietrich Date: Mon, 15 Jan 2018 23:03:33 -0800 Subject: [PATCH 61/71] Update README.md Updated readme to include documented variables recently added to slack script --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c98aa4..5dfd748 100644 --- a/README.md +++ b/README.md @@ -154,10 +154,18 @@ e.g. `SLACK_WEBHOOK_URL https://hooks.slack.com/services/XXXXXXXXX/XXXXXXXXX/XXx A public or private channel in Slack, e.g. `SLACK_CHANNEL #project-name-bots` +`SLACK_USER` + +The username the message should post to Slack as. Defaults to `Docksal CI` + +`SLACK_ICON` + +The icon the message should use to accompany the message: Defaults to `:desktop_computer:` + ### Usage ```bash -slack 'message' ['#channel'] ['webhook_url'] +slack 'message' ['#channel'] ['webhook_url'] ['slack_user'] ['slack_icon'] ``` Channel and webhook url can be passed via environment variables. See above. From 41f379950b88dd7dd68339e2b1563ae7533f5f44 Mon Sep 17 00:00:00 2001 From: Dima Nikitin Date: Tue, 30 Jan 2018 13:23:58 -0500 Subject: [PATCH 62/71] Adding patch package. --- base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/base/Dockerfile b/base/Dockerfile index d61f827..0b619e0 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -8,6 +8,7 @@ RUN apk add --update --no-cache \ openssh \ rsync \ sudo \ + patch \ && rm -rf /var/cache/apk/* ARG GLIBC=2.25-r0 From 594b319baee126ccf0e292ee4bc4ae3b55db241e Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Thu, 15 Mar 2018 13:29:12 -0700 Subject: [PATCH 63/71] Bump versions - Alpine Linux 3.7 - GLIBC 2.27-r0 - Docker Compose 1.19.0 --- base/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index 0b619e0..ee468a9 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.6 +FROM alpine:3.7 RUN apk add --update --no-cache \ bash \ @@ -11,7 +11,7 @@ RUN apk add --update --no-cache \ patch \ && rm -rf /var/cache/apk/* -ARG GLIBC=2.25-r0 +ARG GLIBC=2.27-r0 RUN \ # Install glibc libraries needed for docker-compose apk update && apk add --no-cache openssl ca-certificates && \ @@ -22,7 +22,7 @@ RUN \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib ARG DOCKER_VERSION=17.09.1-ce -ARG DOCKER_COMPOSE_VERSION=1.18.0 +ARG DOCKER_COMPOSE_VERSION=1.19.0 RUN \ # Install docker cli curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" && \ From f04ff79c693c270042b2fab6ce9b4d069525e4f5 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 19 Mar 2018 15:25:24 -0700 Subject: [PATCH 64/71] Use docker v17.09.0-ce That's what is used in Docksal for now. --- base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/Dockerfile b/base/Dockerfile index ee468a9..2997128 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -21,7 +21,7 @@ RUN \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib -ARG DOCKER_VERSION=17.09.1-ce +ARG DOCKER_VERSION=17.09.0-ce ARG DOCKER_COMPOSE_VERSION=1.19.0 RUN \ # Install docker cli From 755c9a079504a43c9ce0ab253b950dbc90465563 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 19 Mar 2018 15:28:50 -0700 Subject: [PATCH 65/71] Dockerfile refactoring - use ; instead of && - indent command arguments --- base/Dockerfile | 52 ++++++++++++++++++++++++------------------------- php/Dockerfile | 52 ++++++++++++++++++++++++------------------------- 2 files changed, 52 insertions(+), 52 deletions(-) diff --git a/base/Dockerfile b/base/Dockerfile index 2997128..eb3d1e6 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -1,58 +1,58 @@ FROM alpine:3.7 RUN apk add --update --no-cache \ - bash \ - curl \ - git \ - jq \ - openssh \ - rsync \ - sudo \ - patch \ - && rm -rf /var/cache/apk/* + bash \ + curl \ + git \ + jq \ + openssh \ + rsync \ + sudo \ + patch; \ + rm -rf /var/cache/apk/*; ARG GLIBC=2.27-r0 RUN \ # Install glibc libraries needed for docker-compose - apk update && apk add --no-cache openssl ca-certificates && \ - curl -L https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub && \ - curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC}/glibc-${GLIBC}.apk && \ - apk add --no-cache glibc-${GLIBC}.apk && rm glibc-${GLIBC}.apk && \ - ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ - ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib + apk update && apk add --no-cache openssl ca-certificates; \ + curl -L https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub; \ + curl -L -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC}/glibc-${GLIBC}.apk; \ + apk add --no-cache glibc-${GLIBC}.apk && rm glibc-${GLIBC}.apk; \ + ln -s /lib/libz.so.1 /usr/glibc-compat/lib/; \ + ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib; ARG DOCKER_VERSION=17.09.0-ce ARG DOCKER_COMPOSE_VERSION=1.19.0 RUN \ # Install docker cli - curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" && \ - tar zxf docker-${DOCKER_VERSION}.tgz && mv docker/docker /usr/local/bin && rm -rf docker-${DOCKER_VERSION}* && \ + curl -sSL -O "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz"; \ + tar zxf docker-${DOCKER_VERSION}.tgz && mv docker/docker /usr/local/bin && rm -rf docker-${DOCKER_VERSION}*; \ # Install docker-compose cli - curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose && \ - chmod +x /usr/local/bin/docker-compose && \ + curl -sSL https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose; \ + chmod +x /usr/local/bin/docker-compose; \ # Install minio client (mc) - curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && \ + curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc; \ chmod +x /usr/local/bin/mc ENV AGENT_USER=agent ENV AGENT_HOME=/home/agent RUN \ # Create a non-root user with access to sudo - adduser -h $AGENT_HOME -s /bin/bash -D $AGENT_USER && \ - echo "$AGENT_USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + adduser -h $AGENT_HOME -s /bin/bash -D $AGENT_USER; \ + echo "$AGENT_USER ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; # All further RUN commands will run as the "agent" user USER $AGENT_USER RUN \ - mkdir -p $AGENT_HOME/.ssh && \ - mkdir -p $AGENT_HOME/build + mkdir -p $AGENT_HOME/.ssh; \ + mkdir -p $AGENT_HOME/build; COPY bin /usr/local/bin COPY config/.ssh/config $AGENT_HOME/.ssh/config # Fix permissions after COPY (could use COPY --chown, but will still need to run chmod) RUN \ - sudo chown $AGENT_USER:$AGENT_USER $AGENT_HOME/.ssh/config && \ - sudo chmod 600 $AGENT_HOME/.ssh/config + sudo chown $AGENT_USER:$AGENT_USER $AGENT_HOME/.ssh/config; \ + sudo chmod 600 $AGENT_HOME/.ssh/config; WORKDIR $AGENT_HOME/build diff --git a/php/Dockerfile b/php/Dockerfile index ddb38ab..8e57249 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -4,21 +4,21 @@ FROM docksal/ci-agent:base USER root RUN apk add --update --no-cache \ - php7 \ - php7-ctype \ - php7-curl \ - php7-dom \ - php7-json \ - php7-mbstring \ - php7-openssl \ - php7-phar \ - php7-simplexml \ - php7-tokenizer \ - php7-xml \ - php7-xmlreader \ - php7-xmlwriter \ - php7-zlib \ - && rm -rf /var/cache/apk/* + php7 \ + php7-ctype \ + php7-curl \ + php7-dom \ + php7-json \ + php7-mbstring \ + php7-openssl \ + php7-phar \ + php7-simplexml \ + php7-tokenizer \ + php7-xml \ + php7-xmlreader \ + php7-xmlwriter \ + php7-zlib; \ + rm -rf /var/cache/apk/*; ENV COMPOSER_VERSION=1.5.1 ENV DRUSH_VERSION 8.1.13 @@ -26,15 +26,15 @@ ENV DRUPAL_CONSOLE_VERSION 1.0.1 ENV WPCLI_VERSION 1.3.0 RUN \ # Composer - curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer && \ + curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer; \ # Drush 8 (default) - curl -sSL "https://github.com/drush-ops/drush/releases/download/${DRUSH_VERSION}/drush.phar" -o /usr/local/bin/drush && \ + curl -sSL "https://github.com/drush-ops/drush/releases/download/${DRUSH_VERSION}/drush.phar" -o /usr/local/bin/drush; \ # Drupal Console - curl -sSL "https://github.com/hechoendrupal/drupal-console-launcher/releases/download/${DRUPAL_CONSOLE_VERSION}/drupal.phar" -o /usr/local/bin/drupal && \ + curl -sSL "https://github.com/hechoendrupal/drupal-console-launcher/releases/download/${DRUPAL_CONSOLE_VERSION}/drupal.phar" -o /usr/local/bin/drupal; \ # Install wp-cli - curl -sSL "https://github.com/wp-cli/wp-cli/releases/download/v${WPCLI_VERSION}/wp-cli-${WPCLI_VERSION}.phar" -o /usr/local/bin/wp && \ + curl -sSL "https://github.com/wp-cli/wp-cli/releases/download/v${WPCLI_VERSION}/wp-cli-${WPCLI_VERSION}.phar" -o /usr/local/bin/wp; \ # Make all binaries executable - chmod +x /usr/local/bin/* + chmod +x /usr/local/bin/*; # Switch back to user USER $AGENT_USER @@ -42,15 +42,15 @@ USER $AGENT_USER ENV PATH $PATH:$AGENT_HOME/.composer/vendor/bin RUN \ # Add composer bin directory to PATH - echo "\n"'PATH="$PATH:$AGENT_HOME/.composer/vendor/bin"' >> $AGENT_HOME/.profile && \ + echo "\n"'PATH="$PATH:$AGENT_HOME/.composer/vendor/bin"' >> $AGENT_HOME/.profile; \ # Drush modules - drush dl registry_rebuild --default-major=7 --destination=$AGENT_HOME/.drush && \ - drush cc drush && \ + drush dl registry_rebuild --default-major=7 --destination=$AGENT_HOME/.drush; \ + drush cc drush; \ # Drupal Coder w/ a matching version of PHP_CodeSniffer - composer global require drupal/coder && \ - phpcs --config-set installed_paths $AGENT_HOME/.composer/vendor/drupal/coder/coder_sniffer && \ + composer global require drupal/coder; \ + phpcs --config-set installed_paths $AGENT_HOME/.composer/vendor/drupal/coder/coder_sniffer; \ # Cleanup - composer clear-cache + composer clear-cache; # Cannot use variables with COPY --chown. See https://github.com/moby/moby/issues/35018 COPY --chown=agent:agent config/.drush $AGENT_HOME/.drush From 841dccfbe732ce2718c25532d362ef1f602716c6 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 19 Mar 2018 15:33:33 -0700 Subject: [PATCH 66/71] Add php7-gd. Fixes #22 --- php/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/php/Dockerfile b/php/Dockerfile index 8e57249..ea74f05 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -8,6 +8,7 @@ RUN apk add --update --no-cache \ php7-ctype \ php7-curl \ php7-dom \ + php7-gd \ php7-json \ php7-mbstring \ php7-openssl \ From 53b069c7c07a0840f861fcb1130c5f725bc53dbf Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 19 Mar 2018 15:56:48 -0700 Subject: [PATCH 67/71] Cleanup output after each run in tests --- tests/base.bats | 3 +++ tests/php.bats | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/tests/base.bats b/tests/base.bats index dda52af..3a3e402 100755 --- a/tests/base.bats +++ b/tests/base.bats @@ -24,14 +24,17 @@ teardown() { run make exec COMMAND="docker --version" [[ "$status" == 0 ]] echo "$output" | grep "Docker version" + unset output run make exec COMMAND="docker-compose --version" [[ "$status" == 0 ]] echo "$output" | grep "docker-compose version" + unset output run make exec COMMAND="mc --help" [[ "$status" == 0 ]] echo "$output" | grep "VERSION" + unset output ### Cleanup ### make clean diff --git a/tests/php.bats b/tests/php.bats index 5544c27..df38a69 100755 --- a/tests/php.bats +++ b/tests/php.bats @@ -19,6 +19,7 @@ teardown() { run bats tests/base.bats [[ "$status" == 0 ]] + unset output } @test "Check binaries" { @@ -31,22 +32,27 @@ teardown() { run make exec COMMAND="php --version" [[ "$status" == 0 ]] echo "$output" | grep "PHP" + unset output run make exec COMMAND="composer --version" [[ "$status" == 0 ]] echo "$output" | grep "Composer version" + unset output run make exec COMMAND="drush --version" [[ "$status" == 0 ]] echo "$output" | grep "Drush Version" + unset output run make exec COMMAND="drupal --version" [[ "$status" == 0 ]] echo "$output" | grep "Drupal Console Launcher" + unset output run make exec COMMAND="wp --version" [[ "$status" == 0 ]] echo "$output" | grep "WP-CLI" + unset output ### Cleanup ### make clean From 30818696771c2c0ca158ad5b325bd3f687af1557 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 19 Mar 2018 15:57:04 -0700 Subject: [PATCH 68/71] Added PHP module test case --- tests/php-modules.txt | 28 ++++++++++++++++++++++++++++ tests/php.bats | 15 +++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/php-modules.txt diff --git a/tests/php-modules.txt b/tests/php-modules.txt new file mode 100644 index 0000000..5b03813 --- /dev/null +++ b/tests/php-modules.txt @@ -0,0 +1,28 @@ +[PHP Modules] +Core +ctype +curl +date +dom +filter +gd +hash +json +libxml +mbstring +openssl +pcre +Phar +readline +Reflection +SimpleXML +SPL +standard +tokenizer +xml +xmlreader +xmlwriter +zlib + +[Zend Modules] + diff --git a/tests/php.bats b/tests/php.bats index df38a69..5dc1e76 100755 --- a/tests/php.bats +++ b/tests/php.bats @@ -57,3 +57,18 @@ teardown() { ### Cleanup ### make clean } + +@test "Check PHP modules" { + [[ $SKIP == 1 ]] && skip + + ### Setup ### + make start + + # Check PHP modules + run bash -c "docker exec '${NAME}' php -m | diff tests/php-modules.txt -" + [[ ${status} == 0 ]] + unset output + + ### Cleanup ### + make clean +} From 86e4cdba1f15b6cdb632ac7705ff264960e349f5 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 19 Mar 2018 17:15:48 -0700 Subject: [PATCH 69/71] Switch to nip.io instead of xip.io New, less used (for now), more reliable (hopefully) --- base/bin/build-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/bin/build-env b/base/bin/build-env index 313aca3..ecbdb15 100755 --- a/base/bin/build-env +++ b/base/bin/build-env @@ -127,8 +127,8 @@ sandbox_server_env () export COMPOSE_PROJECT_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" export DOCKER_STACK_NAME="$REPO_NAME_SAFE-$BRANCH_NAME_SAFE" - # Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with xip.io - export DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.xip.io}" + # Use DOCKSAL_HOST (domain name) if set, otherwise use DOCKSAL_HOST_IP (IP) with nip.io + export DOCKSAL_HOST="${DOCKSAL_HOST:-$DOCKSAL_HOST_IP.nip.io}" sed -i "s/HostName DOCKSAL_HOST/HostName $DOCKSAL_HOST/g" $HOME/.ssh/config # Use ubuntu as the user by default From 2e011870abcde5b301ece875c2e2778d940e4276 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 19 Mar 2018 17:16:40 -0700 Subject: [PATCH 70/71] Version bumps - Composer v1.6.3 - Drush v8.1.16 - Drupal Console v1.7.0 - Wordpress CLI v 1.5.0 --- php/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/php/Dockerfile b/php/Dockerfile index ea74f05..cd22a6d 100644 --- a/php/Dockerfile +++ b/php/Dockerfile @@ -21,10 +21,10 @@ RUN apk add --update --no-cache \ php7-zlib; \ rm -rf /var/cache/apk/*; -ENV COMPOSER_VERSION=1.5.1 -ENV DRUSH_VERSION 8.1.13 -ENV DRUPAL_CONSOLE_VERSION 1.0.1 -ENV WPCLI_VERSION 1.3.0 +ENV COMPOSER_VERSION=1.6.3 +ENV DRUSH_VERSION 8.1.16 +ENV DRUPAL_CONSOLE_VERSION 1.7.0 +ENV WPCLI_VERSION 1.5.0 RUN \ # Composer curl -sSL "https://github.com/composer/composer/releases/download/${COMPOSER_VERSION}/composer.phar" -o /usr/local/bin/composer; \ From 71128e9c9fd4d1e16180ca8ea4d21f2a689ba021 Mon Sep 17 00:00:00 2001 From: Leonid Makarov Date: Mon, 19 Mar 2018 17:16:59 -0700 Subject: [PATCH 71/71] README updates --- README.md | 61 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 5dfd748..0068339 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ A thin agent used to provision Docksal Sandboxes on a remote Docker host. Supported CI providers: -- Bitbucket Pipelines -- CircleCI +- Bitbucket Pipelines (with build status integration) +- CircleCI (with build status integration) - GitLab This image(s) is part of the [Docksal](http://docksal.io) image library. @@ -23,38 +23,37 @@ Use cases: - enhanced pull request review experience - demos -URLs to sandbox environments can be found in the build logs and can also be published to a Slack channel. +Build status updates (and sandbox URLs) can be posted to Github and Bitbucket via respective build status APIs. +URLs to sandbox environments can also be published to a Slack channel. ## Image variants and versions ### Stable -- `docksal/ci-agent:base` - basic (bash, curl, git, etc.), latest version -- `docksal/ci-agent:php` - basic + php stack tools (composer, drush, drupal console, wp-cli, etc.), latest version -- `docksal/ci-agent:1.0-base` - basic, specific stable version -- `docksal/ci-agent:1.0-php` - php, specific stable version +- `base` - basic (bash, curl, git, etc.), latest version +- `php`, `latest` - basic + php stack tools (composer, drush, drupal console, wp-cli, etc.), latest version +- `1.2-base` - basic, a specific stable version +- `1.2-php` - php, a specific stable version ### Development -- `docksal/ci-agent:edge-base` - base, latest development version -- `docksal/ci-agent:edge-php`, php, latest development version +- `edge-base` - base, latest development version +- `edge-php`, php, latest development version ## Global Configuration +### Required + The following required variables are usually configured at the organization level. This way, all project repos will have access to them. They can as well be configured at the repo level. `DOCKSAL_HOST` or `DOCKSAL_HOST_IP` -The address of the remote Docksal host, which will be hosting sandboxes. Configure one of the other. +The address of the remote Docksal host, which is hosting sandboxes. Configure one of the other. If using `DOCKSAL_HOST`, make sure the domain is configured as a wildcard DNS entry. -If using `DOCKSAL_HOST_IP`, the agent will use `xip.io` for dynamic wildcard domain names for sandboxes. - -`DOCKSAL_HOST_USER` - -The user's name that should have access to the remote Docksal host. Defaults to `ubuntu`. +If using `DOCKSAL_HOST_IP`, the agent will use `nip.io` for dynamic wildcard domain names for sandboxes. `DOCKSAL_HOST_SSH_KEY` @@ -68,6 +67,18 @@ A secondary SSH key (base64 encoded as well), which can be used for deployments on the agent. E.g. cloning/pushing a repo, running commands over SSH on a remote deployment environment. + +### Optional + +`DOCKSAL_DOMAIN` + +Can be used to set the base URL for sandbox builds (defaults to `DOCKSAL_HOST` if not set), individually from `DOCKSAL_HOST`. +This is useful when working with CDNs/ELBs/WAFs/etc (when `DOCKSAL_DOMAIN` is different from the `DOCKSAL_HOST`) + +`DOCKSAL_HOST_USER` + +The user's name that should have access to the remote Docksal host. Defaults to `ubuntu`. + `REMOTE_BUILD_BASE` The directory location on the remote server where the repositories should be cloned down to and built. @@ -92,6 +103,12 @@ The user's email to perform Git operations as. Defaults to `ci@docksal.io` The user's name to perform Git operations as. Defaults to `Docksal CI` +`DOCKSAL_HOST_TUNNEL` + +If not empty, `localhost:2374` in the agent is mapped to `docker.sock` on the remote `DOCKSAL_HOST` via a secure SSH tunnel. +The agent can then run `docker` commands against the remote `DOCKSAL_HOST`. + + Other features and integrations are usually configured at the repo level. See below. @@ -102,6 +119,20 @@ file into the project repo and adjust as necessary. For CircleCI, copy the example [config.yml](examples/.circleci/config.yml) file into the project repo and adjust as necessary. +## Build environment variables + +The following environment variables are available to build scripts. +They are derived from respective Bitbucket Pipelines, Circle CI and GitLab CI build variables. + +- `GIT_REPO_OWNER` - git repo machine owner/slug name +- `GIT_REPO_NAME` - git repo machine name +- `GIT_REPO_URL` - git repo URL +- `GIT_BRANCH_NAME` - git branch name +- `GIT_COMMIT_HASH` - git commit hash +- `GIT_PR_NUMBER` - git pull request / merge request number +- `GIT_REPO_SERVICE` - `github`, `bitbucket` or `gitlab` (makes sense mostly for CircleCI) + + ## Feature: Basic HTTP Auth Protect sandboxes from public access using Basic HTTP authentication.