Skip to content

Commit df9bef0

Browse files
krivardkrivard
authored
Merge pull request #936 from cmu-delphi/ds/dev-tools
Add bootstrap installer and a Docker controller Makefile
2 parents a3cf65b + 75dc023 commit df9bef0

File tree

5 files changed

+192
-108
lines changed

5 files changed

+192
-108
lines changed

‎dev/local/.dockerignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Ignore everything by default
2+
*
3+
# Don't ignore repos dir
4+
!repos
5+
# Ignore everything to do with git
6+
**/*.git

‎dev/local/Makefile

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# Docker control panel for delphi-epidata development.
2+
#
3+
# Usage: make <command> [pdb=1] [test=<test-subdir>]
4+
#
5+
# Assumes you have installed your environment using
6+
# delphi-epidata/dev/local/install.sh.
7+
#
8+
# Checks for the delphi-net bridge and creates if it doesn't exist.
9+
#
10+
# Creates all prereq images (delphi_database, delphi_python) only if they don't
11+
# exist. If you need to rebuild a prereq, you're probably doing something
12+
# complicated, and can figure out the rebuild command on your own.
13+
#
14+
#
15+
# Commands:
16+
#
17+
# web: Stops currently-running delphi_web_epidata instances, if any.
18+
# Rebuilds delphi_web_epidata image.
19+
# Runs image in the background and pipes stdout to a log file.
20+
#
21+
# db: Stops currently-running delphi_database_epidata instances, if any.
22+
# Rebuilds delphi_database_epidata image.
23+
# Runs image in the background and pipes stdout to a log file.
24+
# Blocks until database is ready to receive connections.
25+
#
26+
# python: Rebuilds delphi_web_python image. You shouldn't need to do this
27+
# often; only if you are installing a new environment, or have
28+
# made changes to delphi-epidata/dev/docker/python/Dockerfile.
29+
#
30+
# all: Runs the commands 'web' 'db' and 'python'.
31+
#
32+
# test: Runs test and integrations in delphi-epidata. If test
33+
# optional arg is provided, then only the tests in that subdir
34+
# are run.
35+
#
36+
# clean: Cleans up dangling Docker images.
37+
#
38+
#
39+
# Optional arguments:
40+
# pdb=1 Drops you into debug mode upon test failure, if running tests.
41+
# test= Only runs tests in the directories provided here, e.g.
42+
# repos/delphi/delphi-epidata/tests/acquisition/covidcast
43+
44+
45+
# Set optional argument defaults
46+
ifdef pdb
47+
override pdb=--pdb
48+
else
49+
pdb=
50+
endif
51+
52+
ifndef test
53+
test=repos/delphi/delphi-epidata/tests repos/delphi/delphi-epidata/integrations
54+
endif
55+
56+
SHELL:=/bin/sh
57+
58+
# Get the Makefile's absolute path: https://stackoverflow.com/a/324782/4784655
59+
# (if called from a symlink, the path is the location of the symlink)
60+
CWD:=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
61+
NOW:=$(shell date "+%Y-%m-%d")
62+
LOG_WEB:=delphi_web_epidata_$(NOW).log
63+
LOG_DB:=delphi_database_epidata_$(NOW).log
64+
WEB_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_web_epidata')
65+
DATABASE_CONTAINER_ID:=$(shell docker ps -q --filter 'name=delphi_database_epidata')
66+
67+
68+
.PHONY=web
69+
web:
70+
@# Stop container if running
71+
@if [ $(WEB_CONTAINER_ID) ]; then\
72+
docker stop $(WEB_CONTAINER_ID);\
73+
fi
74+
75+
@# Setup virtual network if it doesn't exist
76+
@docker network ls | grep delphi-net || docker network create --driver bridge delphi-net
77+
78+
@# Build the web_epidata image
79+
@cd repos/delphi/delphi-epidata;\
80+
docker build -t delphi_web_epidata -f ./devops/Dockerfile .;\
81+
cd ../../../
82+
83+
@# Run the web server
84+
@docker run --rm -p 127.0.0.1:10080:80 \
85+
--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
86+
--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" \
87+
--network delphi-net --name delphi_web_epidata \
88+
delphi_web_epidata >$(LOG_WEB) 2>&1 &
89+
90+
.PHONY=db
91+
db:
92+
@# Stop container if running
93+
@if [ $(DATABASE_CONTAINER_ID) ]; then\
94+
docker stop $(DATABASE_CONTAINER_ID);\
95+
fi
96+
97+
@# Only build prereqs if we need them
98+
@docker images delphi_database | grep delphi || \
99+
docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
100+
101+
@# Build the database_epidata image
102+
@docker build -t delphi_database_epidata \
103+
-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
104+
105+
@# Run the database
106+
@docker run --rm -p 127.0.0.1:13306:3306 \
107+
--network delphi-net --name delphi_database_epidata \
108+
delphi_database_epidata >$(LOG_DB) 2>&1 &
109+
110+
@# Block until DB is ready
111+
@while true; do \
112+
sed -n '/Temporary server stopped/,/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \
113+
tail -1 $(LOG_DB); \
114+
sleep 1; \
115+
done
116+
117+
.PHONY=py
118+
py:
119+
@# Build the python image
120+
@docker build -t delphi_python \
121+
-f repos/delphi/operations/dev/docker/python/Dockerfile .
122+
123+
@docker build -t delphi_web_python \
124+
-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
125+
126+
.PHONY=all
127+
all: web db py
128+
129+
.PHONY=test
130+
test:
131+
@docker run -i --rm --network delphi-net \
132+
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
133+
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
134+
--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
135+
--env "FLASK_SECRET=abc" \
136+
delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test) | tee test_output_$(NOW).log
137+
138+
.PHONY=clean
139+
clean:
140+
@docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1

‎dev/local/epidata-refresh.sh

Lines changed: 0 additions & 104 deletions
This file was deleted.

‎dev/local/install.sh

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,45 @@
11
#!/bin/bash
2+
# Bootstrap delphi-epidata development
3+
#
4+
# Downloads the repos needed for local delphi-epidata development into current dir
5+
# and provides a Makefile with Docker control commands.
6+
#
7+
# Creates the directory structure:
8+
#
9+
# driver/
10+
# .dockerignore
11+
# Makefile
12+
# repos/
13+
# delphi/
14+
# operations/
15+
# delphi-epidata/
16+
# utils/
17+
# flu-contest/
18+
# nowcast/
19+
# github-deploy-repo/
20+
# undefx/
21+
# py3tester/
22+
# undef-analysis/
23+
#
24+
# Leaves you in driver, the main workdir.
25+
#
226

3-
mkdir -p driver/repos/delphi driver-logs/delphi_database_epidata driver-logs/delphi_web_epidata
27+
28+
mkdir -p driver/repos/delphi
429
cd driver/repos/delphi
530
git clone https://github.com/cmu-delphi/operations
631
git clone https://github.com/cmu-delphi/delphi-epidata
732
git clone https://github.com/cmu-delphi/utils
33+
git clone https://github.com/cmu-delphi/flu-contest
34+
git clone https://github.com/cmu-delphi/nowcast
35+
git clone https://github.com/cmu-delphi/github-deploy-repo
36+
cd ../../
37+
38+
mkdir -p repos/undefx
39+
cd repos/undefx
40+
git clone https://github.com/undefx/py3tester
41+
git clone https://github.com/undefx/undef-analysis
842
cd ../../
9-
ln -s repos/delphi/delphi-epidata/dev/local/epidata-refresh.sh
43+
44+
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
45+
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore

‎docs/epidata_development.md

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,16 @@ nav_order: 4
77

88
## Quickstart
99

10+
In the directory where you want to work run the following
11+
1012
```
1113
$ curl "https://raw.githubusercontent.com/cmu-delphi/delphi-epidata/dev/dev/local/install.sh" | bash
12-
$ cd driver
13-
$ [sudo] ./epidata-refresh.sh database web python
14+
$ [sudo] make all
15+
$ [sudo] make test
16+
# To drop into debugger on error
17+
$ [sudo] make test pdb=1
18+
# To test only a subset of tests
19+
$ [sudo] make test test=repos/delphi/delphi-epidata/integrations/acquisition
1420
```
1521
(sudo requirement depends on your Docker installation and operating system)
1622

0 commit comments

Comments
 (0)