Merge pull request #24 from docksal/develop

Release v1.3.0
docksal · May 31, 2018 · adda6c4 · adda6c4
2 parents d827af9 + 2d0ccca
commit adda6c4
Show file tree

Hide file tree

Showing 8 changed files with 165 additions and 51 deletions.
diff --git a/README.md b/README.md
@@ -81,7 +81,7 @@ The user's name that should have access to the remote Docksal host. Defaults to
 
 `REMOTE_BUILD_BASE`
 
-The directory location on the remote server where the repositories should be cloned down to and built. 
+The default directory location on the remote server where the repositories should be cloned down to and built. 
 Defaults to `/home/ubuntu/builds`
 
 `GITHUB_TOKEN` and `BITBUCKET_TOKEN`
@@ -114,15 +114,56 @@ Other features and integrations are usually configured at the repo level. See be
 
 ## Project configuration
 
-For Bitbucket Pipelines, copy the example [bitbucket-pipelines.yml](examples/bitbucket-pipelines/bitbucket-pipelines.yml) 
-file into the project repo and adjust as necessary.
+### Bitbucket Pipelines
+
+Here's the most basic configuration for Bitbucket Pipelines. Save it into `bitbucket-pipelines.yml` in your project repo.
+
+```yaml
+image: docksal/ci-agent:php
+
+pipelines:
+  default:
+    - step:
+        script:
+          - source build-env && sandbox-init
+```
+
+For a more advanced example see [bitbucket-pipelines.yml](examples/bitbucket-pipelines/bitbucket-pipelines.yml).
+
+### CircleCI
+
+Here's the most basic configuration for CircleCI. Save it into `.circleci/config.yml` in your project repo.
+
+```yaml
+version: 2
+
+jobs:
+  build:
+    working_directory: /home/agent/build
+    docker:
+      - image: docksal/ci-agent:php
+    steps:
+      - checkout
+      - run: source build-env && sandbox-init
+```
+
+For a more advanced example see [config.yml](examples/.circleci/config.yml).
+
+
+## Build commands
+
+For a complete list of built-in commands see [base/bin](base/bin).
+
+- `build-env` - configures build settings on the agent. Usage: `source build-env` (or `DEBUG=1 source build-env`)
+- `build-init`- initializes the sandbox codebase and settings on the sandbox server. Usage: `build-init`
+- `build-exec` - executes a shell command within the build directory on the sandbox server. Usage: `build-init pwd`
+- `build-notify` - see "Build status notifications" docs below
+- `sandbox-init` - a convenient shortcut to provision a basic sandbox. See [sandbox-init](base/bin/sandbox-init) 
 
-For CircleCI, copy the example [config.yml](examples/.circleci/config.yml) file into the project repo and adjust as necessary.
 
 ## Build environment variables
 
-The following environment variables are available to build scripts.  
-They are derived from respective Bitbucket Pipelines, Circle CI and GitLab CI build variables. 
+The following variables are derived from the respective Bitbucket Pipelines, Circle CI and GitLab CI build variables. 
 
 - `GIT_REPO_OWNER` - git repo machine owner/slug name
 - `GIT_REPO_NAME` - git repo machine name
@@ -132,6 +173,14 @@ They are derived from respective Bitbucket Pipelines, Circle CI and GitLab CI bu
 - `GIT_PR_NUMBER` - git pull request / merge request number
 - `GIT_REPO_SERVICE` - `github`, `bitbucket` or `gitlab` (makes sense mostly for CircleCI)
 
+`REMOTE_BUILD_DIR`
+
+The directory location on the remote server where current build will happen. Defaults to:
+
+```
+${REMOTE_BUILD_BASE}/${REPO_NAME_SAFE}-${BRANCH_NAME_SAFE}
+```
+
 
 ## Feature: Basic HTTP Auth
 

diff --git a/base/bin/build-env b/base/bin/build-env
@@ -10,19 +10,35 @@ set -e # Abort if anything fails
 
 # -------------------- Constants -------------------- #
 
+DEBUG=${DEBUG:-0}  # `DEBUG=1 build-env` to run with debugging turned ON
 BRANCH_NAME_LENGTH_LIMIT=35
 DOCKER_HOST_TUNNEL=localhost:2374
 GIT_USER_EMAIL=${GIT_USER_EMAIL:-ci@docksal.io}
 GIT_USER_NAME=${GIT_USER_NAME:-Docksal CI}
 
 # -------------------- Functions -------------------- #
 
+# Print messages only when debugging in ON
+echo-debug ()
+{
+	# Make sure this is wrapped in an "if" statement or it will through "exit 1" on "false" and fail the script
+	if [[ "$DEBUG" != 0 ]]; then echo "$@"; fi
+}
+
+# Generates a domain name safe string (alphanumeric characters and hyphens).
+# Invalid characters are replaced with hyphens.
+# The resulting string is converted to lowercase, since browsers are doing this automatically.
+safe_string ()
+{
+	(sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}') <<< "$1"
+}
+
 # Initial build environment configuration
 build_env ()
 {
 	# Support for Bitbucket Pipelines
 	if [[ "$BITBUCKET_REPO_SLUG" != "" ]]; then
-		echo "Detected Bitbucket Pipelines build environment"
+		echo-debug "Detected Bitbucket Pipelines build environment"
 		export BITBUCKETCI="true"
 		export GIT_REPO_SERVICE="bitbucket"
 		export GIT_REPO_OWNER="$BITBUCKET_REPO_OWNER"
@@ -36,7 +52,7 @@ build_env ()
 
 	# Support for CircleCI 2.0
 	if [[ "$CIRCLECI" != "" ]]; then
-		echo "Detected CircleCI build environment"
+		echo-debug "Detected CircleCI build environment"
 		export GIT_REPO_OWNER="$CIRCLE_PROJECT_USERNAME"
 		export GIT_REPO_NAME="$CIRCLE_PROJECT_REPONAME"
 		export GIT_REPO_URL="$CIRCLE_REPOSITORY_URL"
@@ -59,7 +75,7 @@ build_env ()
 
 	# Support for GitLab 9.0+
 	if [[ "$GITLAB_CI" != "" ]]; then
-		echo "Detected GitLabCI build environment"
+		echo-debug "Detected GitLabCI build environment"
 		export GIT_REPO_SERVICE="gitlab"
 		export GIT_REPO_OWNER="$CI_PROJECT_NAMESPACE"
 		export GIT_REPO_NAME="$CI_PROJECT_NAME"
@@ -77,15 +93,15 @@ build_env ()
 	fi
 
 	# URL safe branch name
-	BRANCH_NAME_SAFE="$(echo -n "$GIT_BRANCH_NAME" | sed -e 's/[^A-Za-z0-9]/-/g' | awk '{print tolower($0)}')"
+	BRANCH_NAME_SAFE="$(safe_string ${GIT_BRANCH_NAME})"
 	# Trim the branch name if longer than BRANCH_NAME_LENGTH_LIMIT and append the md5sum to keep the branch name unique
 	if (( "${#BRANCH_NAME_SAFE}" > "$BRANCH_NAME_LENGTH_LIMIT" )); then
 		BRANCH_NAME_SAFE="$(echo -n "$BRANCH_NAME_SAFE" | cut -c1-30)-$(echo -n "$BRANCH_NAME_SAFE" | md5sum | cut -c1-4)"
 	fi
 	export BRANCH_NAME_SAFE
 
 	# Trim repo name to 30 characters. If someone has a repo name longer than that, then no mercy for them.
-	export REPO_NAME_SAFE="${GIT_REPO_NAME:0:30}"
+	export REPO_NAME_SAFE="$(safe_string ${GIT_REPO_NAME:0:30})"
 	# Short version of GIT_COMMIT_HASH
 	export COMMIT_HASH_SHORT="${GIT_COMMIT_HASH:0:7}"
 }
@@ -137,7 +153,8 @@ sandbox_server_env ()
 
 	# Allow setting DOCKSAL_DOMAIN individually from DOCKSAL_HOST. Default to DOCKSAL_HOST if not set.
 	# This is useful when working with CDNs/ELBs/WAFs/etc (when DOCKSAL_DOMAIN is different from the DOCKSAL_HOST).
-	export DOCKSAL_DOMAIN="${DOCKSAL_DOMAIN:-$DOCKSAL_HOST}"
+	# Make sure domain name is lowercase
+	export DOCKSAL_DOMAIN="$(echo -n ${DOCKSAL_DOMAIN:-$DOCKSAL_HOST} | awk '{print tolower($0)}')"
 	# NOTE: The length of any one label in the domain name is limited to between 1 and 63 octets.
 	export DOMAIN="$BRANCH_NAME_SAFE.$REPO_NAME_SAFE.$DOCKSAL_DOMAIN"
 
@@ -151,16 +168,16 @@ sandbox_server_env ()
 
 # -------------------- Runtime -------------------- #
 
-echo "Configuring build settings..."
+echo-debug "Configuring build settings..."
 build_env
 
-echo "Configuring ssh access..."
+echo-debug "Configuring ssh access..."
 ssh_init
 
 # Configure git
-echo "Configuring git settings..."
+echo-debug "Configuring git settings..."
 git_env
 
 # Sandbox server settings
-echo "Configuring sandbox server settings..."
+echo-debug "Configuring sandbox server settings..."
 sandbox_server_env
diff --git a/base/bin/build-exec b/base/bin/build-exec
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# Simple wrapper to run commands inside the remote build directory on the sandbox server
+
+ssh docker-host "cd $REMOTE_BUILD_DIR && ($@)"
diff --git a/base/bin/build-init b/base/bin/build-init
@@ -2,21 +2,32 @@
 
 # This script sets up a sandbox environment for the build on the remote docker host.
 
-echo "Setting up remote build environment..."
-ssh docker-host whoami
 # Cleanup
-ssh docker-host "(cd $REMOTE_BUILD_DIR && fin rm -f) || true"
-ssh docker-host "sudo rm -rf $REMOTE_BUILD_DIR; mkdir -p $REMOTE_BUILD_DIR"
+echo "Setting up remote build environment..."
+ssh docker-host "(cd $REMOTE_BUILD_DIR && fin rm -f 2>/dev/null) || true"
+ssh docker-host "sudo rm -rf $REMOTE_BUILD_DIR 2>/dev/null; mkdir -p $REMOTE_BUILD_DIR"
+
+# Note: build-exec = ssh docker-host "cd $REMOTE_BUILD_DIR && ($@)"
+
 # Checkout sources
-ssh docker-host "cd $REMOTE_BUILD_DIR && git clone --branch="$GIT_BRANCH_NAME" --depth 50 $GIT_REPO_URL . && git reset --hard $GIT_COMMIT_HASH && ls -la"
+build-exec "git clone --branch="$GIT_BRANCH_NAME" --depth 50 $GIT_REPO_URL . && git reset --hard $GIT_COMMIT_HASH && ls -la"
+
 # Configure sandbox settings
 echo "Configuring sandbox settings..."
-ssh docker-host "cd $REMOTE_BUILD_DIR && echo COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME | tee -a .docksal/docksal-local.env"
-ssh docker-host "cd $REMOTE_BUILD_DIR && echo VIRTUAL_HOST=$DOMAIN | tee -a .docksal/docksal-local.env"
+build-exec "echo COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME | tee -a .docksal/docksal-local.env"
+build-exec "echo VIRTUAL_HOST=$DOMAIN | tee -a .docksal/docksal-local.env"
+
 # Basic HTTP Auth
 if [[ "${HTTP_USER}" != "" ]] && [[ "${HTTP_PASS}" != "" ]]; then
-	ssh docker-host "cd $REMOTE_BUILD_DIR && echo APACHE_BASIC_AUTH_USER=$HTTP_USER | tee -a .docksal/docksal-local.env"
-	ssh docker-host "cd $REMOTE_BUILD_DIR && echo APACHE_BASIC_AUTH_PASS=$HTTP_PASS | tee -a .docksal/docksal-local.env"
+	echo "Configuring sandbox Basic HTTP Authentication..."
+	build-exec "echo APACHE_BASIC_AUTH_USER=$HTTP_USER | tee -a .docksal/docksal-local.env"
+	build-exec "echo APACHE_BASIC_AUTH_PASS=$HTTP_PASS | tee -a .docksal/docksal-local.env"
+fi
+
+# Pass build secrets to sandbox
+# A "secret" is any environment variable that starts with "SECRET_"
+secrets="$(env | grep 'SECRET_')"
+if [[ "$secrets" != "" ]]; then
+	echo "Passing build secrets to sandbox..."
+	build-exec "echo '$secrets' | tee -a .docksal/docksal-local.env >/dev/null"
 fi
-#echo "Launching fin init..."
-#ssh docker-host "cd $REMOTE_BUILD_DIR && fin init"
diff --git a/base/bin/build-notify b/base/bin/build-notify
@@ -5,6 +5,8 @@
 
 #-------------------------- SETTINGS ----------------------------
 
+DEBUG=${DEBUG:-0}  # `DEBUG=1 build-notify` to run with debugging turned ON
+
 export CONTEXT="ci/docksal"  # Context for notifications
 export URL="http://${DOMAIN}"  # Sandbox URL
 export STATUS_API=${STATUS_API:-1}  # Status API notifications enabled by default
@@ -25,7 +27,7 @@ export PR_COMMENT=${PR_COMMENT:-0}  # PR comment notifications disabled by defau
 github_status_api ()
 {
 	[[ "${GIT_REPO_SERVICE}" != "github" ]] && return
-	[[ "${GITHUB_TOKEN}" == "" ]] && echo "Warning: GITHUB_TOKEN not set" && return
+	[[ "${GITHUB_TOKEN}" == "" ]] && echo "Set GITHUB_TOKEN to enable build status notifications" && return
 
 	local context="$1"
 	local state="$2"
@@ -37,11 +39,14 @@ github_status_api ()
 	PAYLOAD="{\"state\": \"${state}\",\"target_url\": \"${url}\",\"description\": \"${description}\",\"context\": \"${context}\"}"
 	API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/statuses/${GIT_COMMIT_HASH}"
 
-	curl -sS --request POST "${API_URL}" \
+	output=$(curl -sS --request POST "${API_URL}" \
 		--header "Content-Type: application/json" \
 		--header "Accept: application/vnd.github.v3.full+json" \
 		--header "Authorization: token ${GITHUB_TOKEN}" \
-		--data "${PAYLOAD}"
+		--data "${PAYLOAD}")
+
+	# Print curl output in debug mode
+	if [[ "$DEBUG" != 0 ]]; then echo "$output"; fi
 }
 
 # Calls Bitbucket Pipelines status API (commit/branch level status updates)
@@ -57,7 +62,7 @@ github_status_api ()
 bitbucket_status_api ()
 {
 	[[ "${GIT_REPO_SERVICE}" != "bitbucket" ]] && return
-	[[ "${BITBUCKET_TOKEN}" == "" ]] && echo "Warning: BITBUCKET_TOKEN not set" && return
+	[[ "${BITBUCKET_TOKEN}" == "" ]] && echo "Set BITBUCKET_TOKEN to enable build status notifications" && return
 
 	local context="$1"
 	local state="$2"
@@ -69,10 +74,13 @@ bitbucket_status_api ()
 	PAYLOAD="{\"key\": \"${context}\", \"state\": \"${state}\", \"description\": \"${description}\", \"url\": \"${url}\"}"
 	API_URL="https://api.bitbucket.org/2.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/commit/${GIT_COMMIT_HASH}/statuses/build"
 
-	curl -sS --request POST "${API_URL}" \
+	output=$(curl -sS --request POST "${API_URL}" \
 		--header "Content-Type: application/json" \
 		--user "${BITBUCKET_TOKEN}" \
-		--data "${PAYLOAD}"
+		--data "${PAYLOAD}")
+
+	# Print curl output in debug mode
+	if [[ "$DEBUG" != 0 ]]; then echo "$output"; fi
 }
 
 # Posts a comment to a pull request in Github
@@ -81,19 +89,22 @@ github_pr_comment ()
 {
 	[[ "${GIT_REPO_SERVICE}" != "github" ]] && return
 	[[ "${GIT_PR_NUMBER}" == "" ]] && return
-	[[ "${GITHUB_TOKEN}" == "" ]] && echo "Warning: GITHUB_TOKEN not set" && return
+	[[ "${GITHUB_TOKEN}" == "" ]] && echo "Set GITHUB_TOKEN to enable build status notifications" && return
 
 	local comment="$1"
 
 	echo "Posting sandbox URL to Github pull request..."
 	PAYLOAD="{\"body\": \"${comment}\"}"
 	API_URL="https://api.github.com/repos/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/issues/${GIT_PR_NUMBER}/comments"
 
-	curl -sS --request POST "${API_URL}" \
+	output=$(curl -sS --request POST "${API_URL}" \
 		--header "Content-Type: application/json" \
 		--header "Accept: application/vnd.github.v3.full+json" \
 		--header "Authorization: token ${GITHUB_TOKEN}" \
-		--data "${PAYLOAD}"
+		--data "${PAYLOAD}")
+
+	# Print curl output in debug mode
+	if [[ "$DEBUG" != 0 ]]; then echo "$output"; fi
 }
 
 # Posts a comment to a pull request in Bitbucket
@@ -102,7 +113,7 @@ bitbucket_pr_comment ()
 {
 	[[ "${GIT_REPO_SERVICE}" != "bitbucket" ]] && return
 	[[ "${GIT_PR_NUMBER}" == "" ]] && return
-	[[ "${BITBUCKET_TOKEN}" == "" ]] && echo "Warning: BITBUCKET_TOKEN not set" && return
+	[[ "${BITBUCKET_TOKEN}" == "" ]] && echo "Set BITBUCKET_TOKEN to enable build status notifications" && return
 
 	local comment="$1"
 
@@ -111,9 +122,12 @@ bitbucket_pr_comment ()
 	# Posting comments to PRs is only available in BB API 1.0
 	API_URL="https://api.bitbucket.org/1.0/repositories/${GIT_REPO_OWNER}/${GIT_REPO_NAME}/pullrequests/${GIT_PR_NUMBER}/comments"
 
-	curl -sS --request POST "${API_URL}" \
+	output=$(curl -sS --request POST "${API_URL}" \
 		--user "${BITBUCKET_TOKEN}" \
-		--data "${PAYLOAD}"
+		--data "${PAYLOAD}")
+
+	# Print curl output in debug mode
+	if [[ "$DEBUG" != 0 ]]; then echo "$output"; fi
 }
 
 notify_pending ()

diff --git a/base/bin/sandbox-init b/base/bin/sandbox-init
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# This script initializes a sandbox environment using the default settings
+
+echo "Initializing codebase and settings on the sandbox server..."
+build-init
+[[ $? == 0 ]] && build-notify pending || build-notify failure
+
+echo "Initializing sandbox via 'fin init'..."
+build-exec "fin init"
+[[ $? == 0 ]] && build-notify success || build-notify failure