Add support for exposing the docker remote API

- Support exposing the Docker Remote API using TLS when certificates are provided
- Add support for setting availability of the manager nodes on provisioning
- Add timeout to docker join and docker leave commands. Avoid long waiting time in case a host cannot be reached
- Add docker leave command when destroying a manager node, allows for down-scaling the amount of managers
- Fix some issues with docker join bash scripting

README.md

@@ -10,7 +10,7 @@ Terraform module to provision and bootstrap a Docker Swarm mode cluster with mul
 
 ## Requirements
 
-- Terraform >= 0.11.2
+- Terraform >= 0.11.7
 - Digitalocean account / API token with write access
 - SSH Keys added to your DigitalOcean account
 - [jq](https://github.com/stedolan/jq)
@@ -19,18 +19,42 @@ Terraform module to provision and bootstrap a Docker Swarm mode cluster with mul
 
 ```hcl
 module "swarm-cluster" {
-  source          = "github.com/thojkooi/terraform-digitalocean-swarm-managers"
+  source          = "github.com/thojkooi/terraform-digitalocean-swarm-managers?ref=v0.2.0"
   domain          = "do.example.com"
   total_instances = 3
-  do_token        = "${var.do_token}"
   ssh_keys        = [1234, 1235, ...]
+  providers {}
 }
 ```
 
 ### SSH Key
 
 Terraform uses an SSH key to connect to the created droplets in order to issue `docker swarm join` commands. By default this uses `~/.ssh/id_rsa`. If you wish to use a different key, you can modify this using the variable `provision_ssh_key`. You also need to ensure the public key is added to your DigitalOcean account and it's ID is listed in the `ssh_keys` list.
 
+### Exposing the Docker API
+
+You can expose the Docker API to interact with the cluster remotely. This is done by providing a certificate and private key. See the [Docker TLS example](https://github.com/thojkooi/terraform-digitalocean-swarm-managers/tree/master/examples/remote-api-tls).
+
+```hcl
+module "swarm_mode_cluster" {
+  source          = "github.com/thojkooi/terraform-digitalocean-swarm-managers?ref=v0.3.0"
+
+  domain          = "do.example.com"
+  total_instances = 3
+  ssh_keys        = [1234, 1235, ...]
+
+  remote_api_ca          = "${path.module}/certs/ca.pem"
+  remote_api_certificate = "${path.module}/certs/server.pem"
+  remote_api_key         = "${path.module}/certs/server-key.pem"
+
+  size = "s-2vcpu-4gb"
+
+  tags = ["${digitalocean_tag.cluster.id}", "${digitalocean_tag.manager.id}"]
+
+  providers = {}
+}
+```
+
 ### Notes
 
 This module does not set up a firewall or modifies any other security settings. Please configure this by providing user data for the manager nodes. Also set up firewall rules on DigitalOcean for the cluster, to ensure only cluster members can access the internal Swarm ports.

examples/add-workers/main.tf

@@ -29,7 +29,7 @@ module "workers" {
   total_instances = 1
   ssh_keys        = "${var.ssh_keys}"
 
-  manager_public_ip  = "${module.swarm-cluster-managers.ipv4_addresses[0]}"
-  manager_private_ip = "${module.swarm-cluster-managers.ipv4_addresses_private[0]}"
+  manager_public_ip  = "${element(module.swarm-cluster-managers.ipv4_addresses, 0)}"
+  manager_private_ip = "${element(module.swarm-cluster-managers.ipv4_addresses_private, 0)}"
   join_token         = "${module.swarm-cluster-managers.worker_token}"
 }

examples/user-data/main.tf

@@ -16,15 +16,10 @@ resource "digitalocean_tag" "manager" {
   name = "manager"
 }
 
-resource "digitalocean_tag" "worker" {
-  name = "worker"
-}
-
 module "swarm-cluster-managers" {
   source          = "../../"
   total_instances = 1
   domain          = "do.example.com"
-  do_token        = "${var.do_token}"
   ssh_keys        = "${var.ssh_keys}"
   image           = "centos-7-x64"
   provision_user  = "root"

examples/user-data/scripts/install-docker-ce.sh

@@ -14,4 +14,5 @@ yum -y install docker-ce
 
 sleep 1;
 
+systemctl enable docker
 systemctl start docker

main.tf

@@ -1,5 +1,19 @@
-provider "digitalocean" {
-  token = "${var.do_token}"
+data "template_file" "provision_first_manager" {
+  template = "${file("${path.module}/scripts/provision-first-manager.sh")}"
+
+  vars {
+    docker_cmd   = "${var.docker_cmd}"
+    availability = "${var.availability}"
+  }
+}
+
+data "template_file" "provision_manager" {
+  template = "${file("${path.module}/scripts/provision-manager.sh")}"
+
+  vars {
+    docker_cmd   = "${var.docker_cmd}"
+    availability = "${var.availability}"
+  }
 }
 
 resource "digitalocean_droplet" "manager" {
@@ -22,18 +36,83 @@ resource "digitalocean_droplet" "manager" {
     timeout     = "2m"
   }
 
+  provisioner "file" {
+    content     = "${data.template_file.provision_first_manager.rendered}"
+    destination = "/tmp/provision-first-manager.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x /tmp/provision-first-manager.sh",
+      "if [ ${count.index} -eq 0 ]; then /tmp/provision-first-manager.sh ${self.ipv4_address_private}; fi",
+    ]
+  }
+
+  provisioner "remote-exec" {
+    when = "destroy"
+
+    inline = [
+      "timeout 25 docker swarm leave",
+    ]
+
+    on_failure = "continue"
+  }
+}
+
+# Optionally expose Docker API using certificates
+resource "null_resource" "manager_api_access" {
+  count = "${var.remote_api_key == "" || var.remote_api_certificate == "" || var.remote_api_ca == "" ? 0 : var.total_instances}"
+
+  triggers {
+    cluster_instance_ids = "${join(",", digitalocean_droplet.manager.*.id)}"
+    certificate          = "${md5(file("${var.remote_api_certificate}"))}"
+  }
+
+  connection {
+    host        = "${element(digitalocean_droplet.manager.*.ipv4_address, count.index)}"
+    type        = "ssh"
+    user        = "${var.provision_user}"
+    private_key = "${file("${var.provision_ssh_key}")}"
+    timeout     = "2m"
+  }
+
   provisioner "remote-exec" {
     inline = [
-      "while [ ! $(${var.docker_cmd} info) ]; do sleep 2; done",
+      "mkdir -p ~/.docker",
+    ]
+  }
 
-      # TODO: Handle failure during swarm init, only run this if manager node is not in a swarm
-      "if [ ${count.index} -eq 0 ]; then ${var.docker_cmd} swarm init --advertise-addr ${digitalocean_droplet.manager.0.ipv4_address_private}; exit 0; fi",
+  provisioner "file" {
+    source      = "${var.remote_api_ca}"
+    destination = "~/.docker/ca.pem"
+  }
+
+  provisioner "file" {
+    source      = "${var.remote_api_certificate}"
+    destination = "~/.docker/server-cert.pem"
+  }
+
+  provisioner "file" {
+    source      = "${var.remote_api_key}"
+    destination = "~/.docker/server-key.pem"
+  }
+
+  provisioner "file" {
+    source      = "${path.module}/scripts/certs/default.sh"
+    destination = "~/.docker/install_certificates.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x ~/.docker/install_certificates.sh",
+      "~/.docker/install_certificates.sh",
     ]
   }
 }
 
 data "external" "swarm_tokens" {
-  program = ["bash", "${path.module}/scripts/get-swarm-join-tokens.sh"]
+  program    = ["bash", "${path.module}/scripts/get-swarm-join-tokens.sh"]
+  depends_on = ["null_resource.manager_api_access"]
 
   query = {
     host        = "${element(digitalocean_droplet.manager.*.ipv4_address, 0)}"
@@ -42,9 +121,9 @@ data "external" "swarm_tokens" {
   }
 }
 
-#
 resource "null_resource" "bootstrap" {
-  count = "${var.total_instances}"
+  count      = "${var.total_instances}"
+  depends_on = ["null_resource.manager_api_access"]
 
   triggers {
     cluster_instance_ids = "${join(",", digitalocean_droplet.manager.*.id)}"
@@ -58,10 +137,15 @@ resource "null_resource" "bootstrap" {
     timeout     = "2m"
   }
 
+  provisioner "file" {
+    content     = "${data.template_file.provision_manager.rendered}"
+    destination = "/tmp/provision-manager.sh"
+  }
+
   provisioner "remote-exec" {
     inline = [
-      "while [ ! $(${var.docker_cmd} info) ]; do sleep 2; done",
-      "if [ ${count.index} -gt 0 ] && [! ${var.docker_cmd} info | grep -q \"Swarm: active\" ]; then ${var.docker_cmd} swarm join --token ${lookup(data.external.swarm_tokens.result, "manager")} ${element(digitalocean_droplet.manager.*.ipv4_address_private, 0)}:2377; exit 0; fi",
+      "chmod +x /tmp/provision-manager.sh",
+      "/tmp/provision-manager.sh ${digitalocean_droplet.manager.0.ipv4_address_private} ${lookup(data.external.swarm_tokens.result, "manager")}",
     ]
   }
 }

scripts/certs/coreos.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Install certificates for the Docker Remote API
+# Based on:
+# - https://coreos.com/os/docs/latest/customizing-docker.html
+# - https://docs.docker.com/engine/reference/commandline/dockerd/
+# -https://docs.docker.com/engine/security/https/#secure-by-default
+
+sudo systemctl stop docker
+sudo systemctl disable docker
+
+sudo mkdir -p /var/ssl
+sudo mv ~/.docker/{server-cert.pem,server-key.pem,ca.pem} /var/ssl/
+
+sudo cat<<-EOF > /etc/systemd/system/docker-tls-tcp.socket
+[Unit]
+Description=Docker Secured Socket for the API
+
+[Socket]
+ListenStream=2376
+BindIPv6Only=both
+Service=docker.service
+
+[Install]
+WantedBy=sockets.target
+EOF
+
+sudo systemctl enable docker-tls-tcp.socket
+sudo systemctl stop docker
+sudo systemctl start docker-tls-tcp.socket
+
+sudo mkdir -p /etc/systemd/system/docker.service.d
+sudo cat<<-EOF > /etc/systemd/system/docker.service.d/10-tls-verify.conf
+[Service]
+Environment="DOCKER_OPTS=--tlsverify=true --tlscacert=/var/ssl/ca.pem --tlscert=/var/ssl/server-cert.pem --tlskey=/var/ssl/server-key.pem"
+EOF
+
+sudo systemctl daemon-reload
+sudo systemctl restart docker.service

scripts/certs/default.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+sudo mkdir -p /var/ssl
+sudo mv ~/.docker/{server-cert.pem,server-key.pem,ca.pem} /var/ssl/
+
+sudo mkdir -p /etc/systemd/system/docker.service.d
+sudo bash -c 'cat<<-EOF > /etc/systemd/system/docker.service.d/10-tls-verify.conf
+[Service]
+Environment="DOCKER_OPTS=--tlsverify=true --tlscacert=/var/ssl/ca.pem --tlscert=/var/ssl/server-cert.pem --tlskey=/var/ssl/server-key.pem"
+ExecStart=
+ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix://var/run/docker.sock --tlsverify=true --tlscacert=/var/ssl/ca.pem --tlscert=/var/ssl/server-cert.pem --tlskey=/var/ssl/server-key.pem
+EOF'
+
+sudo systemctl daemon-reload
+sudo systemctl restart docker

scripts/get-swarm-join-tokens.sh

@@ -4,17 +4,16 @@
 # https://www.terraform.io/docs/providers/external/data_source.html#processing-json-in-shell-scripts
 # Credits to https://github.com/knpwrs/docker-swarm-terraform for inspiration on how to do this
 
-set -e
 eval "$(jq -r '@sh "HOST=\(.host) USER=\(.user) PRIVATE_KEY=\(.private_key)"')"
 
 # Fetch the manager join token
 MANAGER=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i $PRIVATE_KEY \
-    $USER@$HOST docker swarm join-token manager -q)
+    $USER@$HOST timeout 5 docker swarm join-token manager -q)
 
 # Fetch the worker join token
 WORKER=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i $PRIVATE_KEY \
-    $USER@$HOST docker swarm join-token worker -q)
+    $USER@$HOST timeout 5 docker swarm join-token worker -q)
 
 # Produce a JSON object containing the tokens
-jq -n --arg manager "$MANAGER" --arg worker "$WORKER" \
+jq -n --arg manager "${MANAGER}" --arg worker "$WORKER" \
     '{"manager":$manager,"worker":$worker}'

scripts/provision-first-manager.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+MANAGER_PRIVATE_ADDR=$1
+
+# Wait until Docker is running correctly
+while [ -z "$(${docker_cmd} info | grep CPUs)" ]; do
+  echo Waiting for Docker to start...
+  sleep 2
+done
+
+${docker_cmd} swarm init --advertise-addr $MANAGER_PRIVATE_ADDR

scripts/provision-manager.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+MANAGER_PRIVATE_ADDR=$1
+JOIN_TOKEN=$2
+
+# Wait until Docker is running correctly
+while [ -z "$(${docker_cmd} info | grep CPUs)" ]; do
+  echo Waiting for Docker to start...
+  sleep 2
+done
+
+# Check if we are not already joined into a Swarm
+if [ -z "$(${docker_cmd} info | grep 'Swarm: active')" ]; then
+  # Join cluster
+  ${docker_cmd} swarm join --token $JOIN_TOKEN $MANAGER_PRIVATE_ADDR:2377;
+fi

variables.tf

@@ -1,7 +1,3 @@
-variable "do_token" {
-  description = "DigitalOcean API token with read/write permissions"
-}
-
 variable "domain" {
   description = "Domain name used in droplet hostnames, e.g example.com"
 }
@@ -66,3 +62,20 @@ variable "tags" {
   default     = []
   type        = "list"
 }
+
+variable "availability" {
+  description = "Availability of the node ('active'|'pause'|'drain')"
+  default     = "active"
+}
+
+variable "remote_api_ca" {
+  default = ""
+}
+
+variable "remote_api_key" {
+  default = ""
+}
+
+variable "remote_api_certificate" {
+  default = ""
+}