Skip to content

Commit 41e3c1b

Browse files
henrymollmanhenrymollman
authored
Merge pull request #41 from CodeNow/new-dock-init
New dock init
2 parents 0185ce7 + 3c04329 commit 41e3c1b

File tree

6 files changed

+32
-346
lines changed

6 files changed

+32
-346
lines changed

‎init.sh‎

Lines changed: 19 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2,65 +2,40 @@
22

33
# Entry-point script for dock initialization. Simply includes the `lib/dock.sh`
44
# library and calls the master initialization function.
5-
#
6-
# NOTE This script will automatically update the `lib/` directory before
7-
# the dock is initialized. This means that this script itself will not be
8-
# automatically updated. To do so a new AMI must be baked.
9-
#
105
# @author Ryan Sandor Richards
116

127
export DOCK_INIT_BASE=/opt/runnable/dock-init
13-
export CONSUL_HOSTNAME
148
export HOST_IP=$(hostname -i)
15-
export environment=""
169

1710
if [ -z "${CONSUL_PORT+x}" ]; then
1811
export CONSUL_PORT=8500
1912
else
2013
export CONSUL_PORT
2114
fi
2215

16+
if [ -z "${CONSUL_HOSTNAME+x}" ]; then
17+
export CONSUL_HOSTNAME=10.4.5.144
18+
else
19+
export CONSUL_HOSTNAME
20+
fi
21+
2322
source "${DOCK_INIT_BASE}/lib/consul.sh"
23+
source "${DOCK_INIT_BASE}/lib/aws.sh"
24+
source "${DOCK_INIT_BASE}/lib/dock.sh"
25+
source "${DOCK_INIT_BASE}/lib/container.sh"
2426
source "${DOCK_INIT_BASE}/lib/util/log.sh"
25-
source "${DOCK_INIT_BASE}/lib/util/rollbar.sh"
26-
source "${DOCK_INIT_BASE}/lib/util/backoff.sh"
27-
28-
# Executes a command using an ssh agent with the id_rsa_runnabledock key
29-
# @param $1 action Comand to execute
30-
ssh_execute() {
31-
local action="$1"
32-
ssh-agent bash -c "ssh-add key/id_rsa_runnabledock; $action"
33-
}
34-
35-
# Automatically updates dock-init to the version given in consul, if needed.
36-
# After consul has been updated this executes the main script.
37-
auto_update() {
38-
log::info "Updating dock-init"
39-
consul::connect
40-
41-
log::trace 'Fetching dock-init version from consul...'
42-
local version=$(consul::get '/dock-init/version')
43-
log::info "dock-init version found: $version"
44-
45-
log::trace "moving to dock init base directory ($DOCK_INIT_BASE)"
46-
cd "$DOCK_INIT_BASE"
47-
48-
log::trace "fetching all from repository"
49-
if [[ "$FETCH_ORIGIN_ALL" != "" ]]; then
50-
ssh_execute "git fetch origin $version"
51-
else
52-
ssh_execute "git fetch origin"
53-
fi
54-
55-
log::info "Checking out dock-init version: $version"
56-
ssh_execute "git checkout $version"
57-
}
5827

5928
# Initializes the dock
6029
main() {
61-
source "${DOCK_INIT_BASE}/lib/dock.sh"
62-
dock::init
30+
consul::connect
31+
consul::get_environment
32+
consul::configure_consul_template
33+
dock::generate_certs
34+
aws::get_org_id
35+
dock::set_hostname
36+
dock::set_config_org
37+
container::start
38+
log::info "Init Done!"
6339
}
6440

65-
# Attempt to auto-update then initialize the dock
66-
backoff auto_update && main
41+
main

‎lib/container.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@ source "${DOCK_INIT_BASE}/lib/consul.sh"
77
source "${DOCK_INIT_BASE}/lib/util/backoff.sh"
88
source "${DOCK_INIT_BASE}/lib/util/log.sh"
99
source "${DOCK_INIT_BASE}/lib/util/rollbar.sh"
10+
source "${DOCK_INIT_BASE}/lib/upstart.sh"
1011
source "${DOCK_INIT_BASE}/lib/vault.sh"
1112

1213
# Starts the docker swarm container
1314
container::_start_swarm_container() {
1415
local name="swarm"
15-
local version="$(consul::get ${name}/version)"
16+
local version="1.2.5"
1617

1718
log::info "Starting swarm:${version} container"
1819
local docker_logs
@@ -37,7 +38,7 @@ container::_start_swarm_container() {
3738
# Starts the docker registry container
3839
container::_start_registry_container() {
3940
local name="registry"
40-
local version="$(consul::get ${name}/version)"
41+
local version="2.3.1"
4142
log::info "Starting ${name}:${version} container"
4243

4344
local region="$(consul::get s3/region)"
@@ -131,12 +132,17 @@ container::_start_node_exporter_container() {
131132
# Starts all container services needed for the dock
132133
container::start() {
133134
log::info "Starting container services"
135+
upstart::start_docker
134136
backoff container::_start_registry_container
135137
backoff container::_start_cadvisor_container
136138
backoff container::_start_node_exporter_container
137139

138140
# swarm should be started last so we know everything is up
139141
backoff container::_start_swarm_container
142+
# currently @henrymollman does not understand why restarting swarm works
143+
# but without this line docker-listener will time out getting events
144+
# and the stream will close. this is an intermittent error however
145+
docker restart swarm
140146
}
141147

142148
# Stops all dock container services

‎lib/dock.sh‎

Lines changed: 0 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -9,38 +9,11 @@
99
# @author Bryan Kendall
1010
# @module dock
1111

12-
source "${DOCK_INIT_BASE}/lib/aws.sh"
1312
source "${DOCK_INIT_BASE}/lib/cert.sh"
14-
source "${DOCK_INIT_BASE}/lib/consul.sh"
15-
source "${DOCK_INIT_BASE}/lib/container.sh"
16-
source "${DOCK_INIT_BASE}/lib/upstart.sh"
17-
1813
source "${DOCK_INIT_BASE}/lib/util/backoff.sh"
1914
source "${DOCK_INIT_BASE}/lib/util/log.sh"
2015
source "${DOCK_INIT_BASE}/lib/util/rollbar.sh"
2116

22-
# An "on exit" trap to clean up sensitive keys and files on the dock itself.
23-
# Note that this will have no effect if the `DONT_DELETE_KEYS` environment has
24-
# been set (useful for testing)
25-
dock::cleanup::exit_trap() {
26-
# Delete the keys unless the `DO_NOT_DELETE` flag is set
27-
if [[ "${DONT_DELETE_KEYS}" == "" ]]; then
28-
log::info '[CLEANUP TRAP] Removing Keys'
29-
rm -f "${CERT_PATH}"/ca-key.pem \
30-
"${CERT_PATH}"/pass \
31-
"${DOCK_INIT_BASE}"/consul-resources/template-config.hcl \
32-
"${DOCK_INIT_BASE}"/consul-resources/vault/**/auth-token \
33-
"${DOCK_INIT_BASE}"/consul-resources/vault/**/token-* \
34-
"${DOCK_INIT_BASE}"/key/rollbar.token
35-
fi
36-
}
37-
38-
# Sets the cleanup trap for the entire script
39-
dock::cleanup::set_exit_trap() {
40-
log::info "Setting key cleanup trap"
41-
trap 'dock::cleanup::exit_trap' EXIT
42-
}
43-
4417
# Sets the value of `$ORG_ID` as the org label in the docker configuration
4518
dock::set_config_org() {
4619
log::info "Setting organization id in docker configuration"
@@ -68,60 +41,3 @@ dock::generate_certs() {
6841
backoff dock::generate_certs_backoff
6942
}
7043

71-
# Generates the correct /etc/hosts file for the dock
72-
dock::generate_etc_hosts() {
73-
log::info "Generating /etc/hosts"
74-
75-
rollbar::fatal_trap \
76-
"Dock-Init: Failed to Add Host Registry Entry" \
77-
"Consul-Template was unable to realize the registry template."
78-
79-
local template=''
80-
template+="$DOCK_INIT_BASE/consul-resources/templates/hosts-registry.ctmpl"
81-
template+=":$DOCK_INIT_BASE/hosts-registry.txt"
82-
consul-template \
83-
-config="${DOCK_INIT_BASE}"/consul-resources/template-config.hcl \
84-
-once \
85-
-template="${template}"
86-
87-
rollbar::clear_trap
88-
}
89-
90-
# Sets the correct registry.runnable.com host
91-
dock::set_registry_host() {
92-
local registry_host=$(cat "$DOCK_INIT_BASE/hosts-registry.txt")
93-
log::info "Set registry host: $registry_host"
94-
echo "$registry_host" >> /etc/hosts
95-
}
96-
97-
# Remove docker key file so it generates a unique id
98-
dock::remove_docker_key_file() {
99-
log::info "Removing docker key.json"
100-
rm -f /etc/docker/key.json
101-
}
102-
103-
# Master function for performing all tasks and initializing the dock
104-
dock::init() {
105-
# Setup the exit trap and rollbar
106-
dock::cleanup::set_exit_trap
107-
rollbar::init
108-
109-
# Connect to and configure consul then collect various information we need
110-
consul::connect
111-
consul::get_environment
112-
consul::configure_consul_template
113-
aws::get_org_id
114-
115-
# Now that we have everything we need and consul is ready, initialize the dock
116-
dock::set_hostname
117-
dock::set_config_org
118-
dock::generate_certs
119-
dock::generate_etc_hosts
120-
dock::set_registry_host
121-
dock::remove_docker_key_file
122-
upstart::start
123-
container::start
124-
125-
# Give the all clear message!
126-
log::info "Init Done!"
127-
}

‎lib/upstart.sh‎

Lines changed: 1 addition & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -5,95 +5,9 @@
55
# @author Ryan Sandor Richards
66
# @author Bryan Kendall
77

8-
source "${DOCK_INIT_BASE}/lib/consul.sh"
9-
source "${DOCK_INIT_BASE}/lib/util/backoff.sh"
108
source "${DOCK_INIT_BASE}/lib/util/log.sh"
119
source "${DOCK_INIT_BASE}/lib/util/rollbar.sh"
1210

13-
# Generates upstart scripts for the dock
14-
upstart::generate_scripts() {
15-
log::info "Generating Upstart Scripts"
16-
rollbar::fatal_trap \
17-
"Dock-Init: Failed to Generate Upstart Script" \
18-
"Failed to generate the upstart scripts."
19-
upstart::generate_scripts
20-
rollbar::clear_trap
21-
}
22-
23-
# Configures the template for a given service
24-
# @param $1 name Name of the service
25-
# @param $2 path Path to the servic
26-
upstart::configure_service() {
27-
local name="${1}"
28-
log::trace "Configuring $name"
29-
rollbar::fatal_trap \
30-
"Consul-Template: Failed to Render $name Config" \
31-
"Consule-Template was unable to realize the given template."
32-
33-
local template_path="$DOCK_INIT_BASE/consul-resources/templates/services"
34-
template_path+="/$name.conf.ctmpl"
35-
template_path+=":/etc/init/$name.conf"
36-
37-
consul-template \
38-
-config="$DOCK_INIT_BASE/consul-resources/template-config.hcl" \
39-
-once \
40-
-template="$template_path"
41-
echo "manual" > /etc/init/"$name".override
42-
43-
rollbar::clear_trap
44-
}
45-
46-
# Generates upstart scripts for thoses services that require environment info
47-
# from consul
48-
upstart::generate_scripts() {
49-
log::info "Configuring Upstart Scripts"
50-
upstart::configure_service "charon"
51-
log::trace "Done Generating Upstart Scripts"
52-
}
53-
54-
# Updates a service to the consul version, installs packages, then restarts it.
55-
# @param $1 Name of the service
56-
upstart::upstart_named_service() {
57-
local name="${1}"
58-
local attempt="${2}"
59-
local data='{"attempt":'"${attempt}"'}'
60-
local version="$(consul::get ${name}/version)"
61-
local key_path="$DOCK_INIT_BASE/key/id_rsa_runnabledock"
62-
63-
rollbar::warning_trap \
64-
"$name: Cannot Upstart Services" \
65-
"Attempting to upstart the services and failing." \
66-
"${data}"
67-
68-
log::info "Updating and restarting $name @ $version" &&
69-
cd "/opt/runnable/$name" &&
70-
ssh-agent bash -c "ssh-add $key_path; git fetch origin" &&
71-
git checkout "$version" &&
72-
ssh-agent bash -c "ssh-add $key_path; USERPROFILE=/home/ubuntu npm install --production" &&
73-
service "$name" restart
74-
75-
rollbar::clear_trap
76-
}
77-
78-
# Starts a service installed on the machine.
79-
# @param $1 Name of the service
80-
# @param $2 Attempt number
81-
upstart::upstart_service() {
82-
local name="${1}"
83-
local attempt="${2}"
84-
local data='{"attempt":'"${attempt}"'}'
85-
86-
rollbar::warning_trap \
87-
"$name: Cannot Upstart Service" \
88-
"Attempting to upstart the service and failing." \
89-
"${data}"
90-
91-
log::info "Starting $name"
92-
service "$name" restart
93-
94-
rollbar::clear_trap
95-
}
96-
9711
# Start dockers (due to manual override now set in /etc/init)
9812
upstart::start_docker() {
9913
log::info "Starting Docker"
@@ -105,7 +19,7 @@ upstart::start_docker() {
10519

10620
log::info "Waiting for Docker"
10721
local attempt=1
108-
local timeout=1
22+
local timeout=.5
10923
while [ ! -e /var/run/docker.sock ]
11024
do
11125
log::info "Docker Sock N/A ($attempt)"
@@ -115,52 +29,5 @@ upstart::start_docker() {
11529
rollbar::report_warning "${title}" "${message}" "$data"
11630
sleep $timeout
11731
attempt=$(( attempt + 1 ))
118-
timeout=$(( timeout * 2 ))
11932
done
12033
}
121-
122-
# Upstarts services that are supposed to be running on the dock.
123-
# @param $1 attempt Attempt number.
124-
upstart::upstart_services_with_backoff_params() {
125-
local attempt="${1}"
126-
upstart::upstart_named_service "krain" $attempt
127-
upstart::upstart_named_service "charon" $attempt
128-
upstart::upstart_service "datadog-agent" $attempt
129-
}
130-
131-
# Pulls the latest docker image for the runnable image builder
132-
# @param $1 attempt The current attempt for pulling image builder
133-
upstart::pull_image_builder() {
134-
local attempt="${1}"
135-
local name="image-builder"
136-
local version="$(consul::get $name/version)"
137-
138-
log::info "Pulling image-builder:$version (${attempt})"
139-
docker pull "registry.runnable.com/runnable/image-builder:$version"
140-
141-
if [[ "$?" -gt "0" ]]; then
142-
local data='{"attempt":'"${attempt}"'}'
143-
rollbar::report_warning \
144-
"Dock-Init: Cannot Upstart Services" \
145-
"Attempting to upstart the services and failing." \
146-
"${data}"
147-
return 1
148-
fi
149-
}
150-
151-
# Starts all services needed for the dock
152-
upstart::start() {
153-
log::info "Upstarting dock"
154-
upstart::generate_scripts
155-
upstart::start_docker
156-
backoff upstart::pull_image_builder
157-
backoff upstart::upstart_services_with_backoff_params
158-
}
159-
160-
# Stops all dock services
161-
upstart::stop() {
162-
log::info "Stopping all dock upstart services"
163-
service krain stop
164-
service charon stop
165-
service docker stop
166-
}

0 commit comments

Comments
 (0)