Merge pull request #23 from docksal/develop

Release 1.2.0

.gitignore

@@ -0,0 +1 @@
+env_make

.travis.yml

@@ -0,0 +1,28 @@
+language: generic
+
+services:
+  - docker
+
+env:
+  global:
+    - REPO=docksal/ci-agent
+
+install:
+  - curl -fsSL get.docksal.io | sh
+  - fin version
+  - fin sysinfo
+
+script:
+  # Build base first, then php
+  - make VERSION=base && make test VERSION=base
+  - make VERSION=php && make test VERSION=php
+
+after_success: |
+  # Push images to Docker Hub
+  if [[ "${TRAVIS_PULL_REQUEST}" == "false" ]]; then
+    scripts/push-image.sh base
+    scripts/push-image.sh php
+  fi
+
+after_failure:
+  - make logs

Makefile

@@ -0,0 +1,43 @@
+-include env_make
+
+VERSION ?= base
+
+REPO = docksal/ci-agent
+NAME = ci-agent
+
+.PHONY: build test push shell run start stop logs clean release
+
+build:
+	docker build -t $(REPO):$(VERSION) $(VERSION)
+
+test:
+	IMAGE=$(REPO):$(VERSION) NAME=$(NAME) tests/$(VERSION).bats
+
+push:
+	docker push $(REPO):$(VERSION)
+
+shell: clean
+	docker run --rm --name $(NAME) -it $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) /bin/bash
+
+exec:
+	docker exec $(NAME) $(COMMAND)
+
+run: clean
+	docker run --rm --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION)
+
+start: clean
+	docker run -d --name $(NAME) $(PORTS) $(VOLUMES) $(ENV) $(REPO):$(VERSION) top -b
+
+stop:
+	docker stop $(NAME)
+
+logs:
+	docker logs $(NAME)
+
+clean:
+	docker rm -f $(NAME) || true
+
+release: build
+	make push -e VERSION=$(VERSION)
+
+default: build

README.md

@@ -1,42 +1,59 @@
-# Bitbucket Pipelines Agent Docker image for Docksal CI
+# CI agent Docker image for Docksal Sandboxes
 
-A thin agent used to provision Docksal powered sandboxes on a remote Docker host.
+A thin agent used to provision Docksal Sandboxes on a remote Docker host.  
+
+Supported CI providers:
+
+- Bitbucket Pipelines (with build status integration)
+- CircleCI (with build status integration)
+- GitLab
 
 This image(s) is part of the [Docksal](http://docksal.io) image library.
 
 
-## Docksal CI Sandboxes
+## Docksal Sandboxes
+
+Docksal Sandboxes are continuous integration environments powered by Docksal.  
+They can be provisioned for any git branch and are feature-wise identical to project's local Docksal environment.
 
-For any Docksal powered project enabling Bitbucket Pipelines and dropping [this](bitbucket-pipelines.yml) file 
-into the project repo will enable per-branch sandbox provisioning.
-URLs to sandbox environments can be found in the build logs and can also published to a desired Slack channel.
+Use cases:
+
+- automated testing (full stack)
+- manual testing
+- enhanced pull request review experience
+- demos
+
+Build status updates (and sandbox URLs) can be posted to Github and Bitbucket via respective build status APIs.  
+URLs to sandbox environments can also be published to a Slack channel.  
 
 
 ## Image variants and versions
 
 ### Stable
 
-- `docksal/bitbucket-pipelines-agent` - basic (bash, curl, git)
-- `docksal/bitbucket-pipelines-agent:php` - basic + php stack tools (composer, drush, drupal console, wp-cli, etc)
-- `docksal/bitbucket-pipelines-agent:1.0` - basic, specific version
-- `docksal/bitbucket-pipelines-agent:1.0-php` - php, specific version
+- `base` - basic (bash, curl, git, etc.), latest version
+- `php`, `latest` - basic + php stack tools (composer, drush, drupal console, wp-cli, etc.), latest version
+- `1.2-base` - basic, a specific stable version
+- `1.2-php` - php, a specific stable version
 
 ### Development
 
-- `docksal/bitbucket-pipelines-agent:edge`
-- `docksal/bitbucket-pipelines-agent:edge-php`
+- `edge-base` - base, latest development version
+- `edge-php`, php, latest development version
+
 
-## Configuration
+## Global Configuration
 
-### Global Pipelines variables
+### Required
 
-The agent expects the following required variables to be defined.
+The following required variables are usually configured at the organization level. This way, all project repos will 
+have access to them. They can as well be configured at the repo level.
 
 `DOCKSAL_HOST` or `DOCKSAL_HOST_IP`
 
-The address of the remote Docksal host, which will be hosting sandboxes. Configure one of the other.  
+The address of the remote Docksal host, which is hosting sandboxes. Configure one of the other.  
 If using `DOCKSAL_HOST`, make sure the domain is configured as a wildcard DNS entry.  
-If using `DOCKSAL_HOST_IP`, the agent will use `xip.io` for dynamic wildcard domain names for sandboxes. 
+If using `DOCKSAL_HOST_IP`, the agent will use `nip.io` for dynamic wildcard domain names for sandboxes. 
 
 `DOCKSAL_HOST_SSH_KEY`
 
@@ -46,13 +63,118 @@ tutorial for details.
 
 `CI_SSH_KEY`
 
-A second SSH keys (base64 encoded as well), which can be used for deployments and other remote operations run directly 
-on the agent.  
+A secondary SSH key (base64 encoded as well), which can be used for deployments and other remote operations run directly 
+on the agent.
 E.g. cloning/pushing a repo, running commands over SSH on a remote deployment environment.
 
-### Project level Pipelines variables 
 
-The following environment variables can be configured in the project's Pipelines settings:
+### Optional
+
+`DOCKSAL_DOMAIN`
+
+Can be used to set the base URL for sandbox builds (defaults to `DOCKSAL_HOST` if not set), individually from `DOCKSAL_HOST`.  
+This is useful when working with CDNs/ELBs/WAFs/etc (when `DOCKSAL_DOMAIN` is different from the `DOCKSAL_HOST`)
+
+`DOCKSAL_HOST_USER`
+
+The user's name that should have access to the remote Docksal host. Defaults to `ubuntu`.
+
+`REMOTE_BUILD_BASE`
+
+The directory location on the remote server where the repositories should be cloned down to and built. 
+Defaults to `/home/ubuntu/builds`
+
+`GITHUB_TOKEN` and `BITBUCKET_TOKEN`
+
+Used for access to post sandbox URLs via build status API as well as comments on pull requests.  
+
+For Github, the token can be generated from the [user's account](https://github.com/settings/tokens).  
+Set access to "repo" (http://take.ms/nMqcW).
+
+For Bitbucket, the token can be generated from the user's settings. Instructions on creating an [app password](https://confluence.atlassian.com/bitbucket/app-passwords-828781300.html).  
+Set access to "Repositories: Write", "Pull requests: Write" (http://take.ms/98BG5).  
+When storing the app password it is in the format: `USER:PASSWORD`.
+
+`GIT_USER_EMAIL`
+
+The user's email to perform Git operations as. Defaults to `[email protected]`
+
+`GIT_USER_NAME`
+
+The user's name to perform Git operations as. Defaults to `Docksal CI`
+
+`DOCKSAL_HOST_TUNNEL`
+
+If not empty, `localhost:2374` in the agent is mapped to `docker.sock` on the remote `DOCKSAL_HOST` via a secure SSH tunnel.  
+The agent can then run `docker` commands against the remote `DOCKSAL_HOST`.
+
+
+Other features and integrations are usually configured at the repo level. See below.
+
+
+## Project configuration
+
+For Bitbucket Pipelines, copy the example [bitbucket-pipelines.yml](examples/bitbucket-pipelines/bitbucket-pipelines.yml) 
+file into the project repo and adjust as necessary.
+
+For CircleCI, copy the example [config.yml](examples/.circleci/config.yml) file into the project repo and adjust as necessary.
+
+## Build environment variables
+
+The following environment variables are available to build scripts.  
+They are derived from respective Bitbucket Pipelines, Circle CI and GitLab CI build variables. 
+
+- `GIT_REPO_OWNER` - git repo machine owner/slug name
+- `GIT_REPO_NAME` - git repo machine name
+- `GIT_REPO_URL` - git repo URL
+- `GIT_BRANCH_NAME` - git branch name
+- `GIT_COMMIT_HASH` - git commit hash
+- `GIT_PR_NUMBER` - git pull request / merge request number
+- `GIT_REPO_SERVICE` - `github`, `bitbucket` or `gitlab` (makes sense mostly for CircleCI)
+
+
+## Feature: Basic HTTP Auth
+
+Protect sandboxes from public access using Basic HTTP authentication.
+
+### Configuration
+
+Set the following environment variables at the repo level:
+
+- `HTTP_USER`
+- `HTTP_PASS`
+
+
+## Feature: Build status notifications
+
+This integration allows the agent to post build status updates and sandbox URL via Github/Bitbucket build status API.  
+For CircleCI, it is also possible to enable posting the sandbox URL as a comment in pull requests. 
+
+### Configuration
+
+`GITHUB_TOKEN` or `BITBUCKET_TOKEN` must be configured respectively (either globally or at the repo level). 
+
+### Usage
+
+`build-notify <pending|success|failure>`
+
+Place the triggers right before and right after `fin init` call in your build script, e.g.
+
+```bash
+build-notify pending 
+ssh docker-host "cd $REMOTE_BUILD_DIR && fin init" || ( build-notify failure && exit 1 )
+build-notify success
+```
+
+To enable posting sandbox URLs in comments on pull requests, do `export PR_COMMENT=1` prior to calling `build-notify`
+
+
+## Feature: Slack notifications
+
+This integrations allows the agent to post messages to a given Slack channel.  
+It can be used for notification purposes when a build is started, completed, failed, etc.
+
+### Configuration
 
 `SLACK_WEBHOOK_URL`
 
@@ -63,14 +185,111 @@ e.g. `SLACK_WEBHOOK_URL https://hooks.slack.com/services/XXXXXXXXX/XXXXXXXXX/XXx
 
 A public or private channel in Slack, e.g. `SLACK_CHANNEL #project-name-bots`
 
-## Slack notifications
+`SLACK_USER`
+
+The username the message should post to Slack as. Defaults to `Docksal CI`
+
+`SLACK_ICON`
+
+The icon the message should use to accompany the message: Defaults to `:desktop_computer:`
 
 ### Usage
 
-`slack 'message' ['#channel'] ['webhook_url']`
+```bash
+slack 'message' ['#channel'] ['webhook_url'] ['slack_user'] ['slack_icon']
+```
 
-Channel and webhook url can be passed via environment variables. See below.
+Channel and webhook url can be passed via environment variables. See above. 
 
 ### Limitations
 
 Incoming Webhook integration won't work for private channels, which the owner of the integration does not belong to.
+
+
+## Feature: Build artifact storage
+
+Build artifacts can be stored in an AWS S3 bucket.  
+
+### Configuration
+
+Set the following environment variables at the organization or repo level:
+
+- `AWS_ACCESS_KEY_ID`
+- `AWS_SECRET_ACCESS_KEY`
+- `ARTIFACTS_BUCKET_NAME`
+- `ARTIFACTS_BASE_URL` (optional)
+- `GITHUB_TOKEN` (optional)
+- `BITBUCKET_TOKEN` (optional)
+
+### Usage
+
+To upload artifacts to the configured S3 bucket use the `build-acp` command.
+
+```bash
+build-acp /source/path/
+```
+
+There is no file browsing capability available for private S3 buckets.  
+An `index.html` file is used as the directory index, however it has to be created/generated manually.  
+When uploading a prepared artifacts folder with the `index.html` file in it, add a trailing slash to the source path to
+upload the contents of the source folder vs the folder itself.
+
+You can upload additional folders/files by running the command multiple times.
+
+The optional `destination` argument can be used to define a subdirectory at the destination in the bucket.  
+
+```bash
+build-acp /source/path/ destination/path
+```
+
+### Advanced usage
+
+**Base URL**
+
+The base URL is derived from `ARTIFACTS_BUCKET_NAME` as follows (assuming AWS S3 `us-east-1` region by default):
+
+```
+https://${ARTIFACTS_BUCKET_NAME}.s3.amazonaws.com
+```
+
+It can be overridden via the optional `ARTIFACTS_BASE_URL` configuration variable at the organization/repo level:
+
+```
+ARTIFACTS_BASE_URL = https://artifacts.example.com
+```
+
+**Upload path**
+
+The upload path is unique for each commit and is derived as follows: 
+
+```bash
+${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}-${GIT_COMMIT_HASH}
+```
+
+In certain cases you may want to store build artifacts per branch instead of per commit.  
+To do this, override the `ARTIFACTS_BUCKET_PATH` variable before calling the `build-acp` command: 
+
+```bash
+export ARTIFACTS_BUCKET_PATH="${REPO_NAME_SAFE}/${BRANCH_NAME_SAFE}"
+build-acp my-artifacts/
+```
+
+**Posting build artifact URLs to Bitbucket**
+
+If `BITBUCKET_TOKEN` is set, the URL to the artifacts will be posted back to Bitbucket via 
+[Bitbucket Build Status API](https://blog.bitbucket.org/2015/11/18/introducing-the-build-status-api-for-bitbucket-cloud/).
+
+### Security
+
+If a bucket does not exist, it will be created automatically (with no public access). Existing bucket access permissions 
+are not automatically adjusted. It's up to you whether you want to keep them open or not.
+
+When artifacts are uploaded, the destination artifact folder in the bucket is set to be publicly accessible. 
+Anyone with the direct link will be able to access the artifacts, but will not be able to browse the list of all 
+available artifact folders in the bucket (so long as the bucket itself is set to be private). 
+
+The URL by default includes a git commit hash, which serves as an authentication token (the URL is impossible to guess). 
+This provides a simple yet efficient level of security for artifacts.
+
+To add an additional level of security follow [this guide](https://medium.com/@lmakarov/serverless-password-protecting-a-static-website-in-an-aws-s3-bucket-bfaaa01b8666) 
+to set up username/password access to S3 via CloudFront and Lambda@Edge.