Skip to content

Commit 184a385

Browse files
dshemetovdshemetov
committed
Add Docker controller Makefile and boostrap script
* combine with Katie and Xavier's work in epidata-refresh.sh * simplify directory structure using .dockerignore * add in missing dependencies to install.sh * add in changes reviews from Xavier's review
1 parent a3cf65b commit 184a385

File tree

5 files changed

+170
-109
lines changed

5 files changed

+170
-109
lines changed

‎dev/local/.dockerignore

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Ignore everything by default
2+
*
3+
# Don't ignore repos dir
4+
!repos
5+
# Ignore everything to do with git
6+
**/*.git

‎dev/local/Makefile

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# Docker control panel for delphi-epidata development
2+
#
3+
# Usage: make <command> [pdb=1] [test-subdir=<test-subdir>]
4+
#
5+
# Assumes you have installed your environment using
6+
# delphi-epidata/dev/local/install.sh.
7+
#
8+
# Checks for the delphi-net bridge and creates if it doesn't exist.
9+
#
10+
# Creates all prereq images (delphi_database, delphi_python) only if they don't
11+
# exist. If you need to rebuild a prereq, you're probably doing something
12+
# complicated, and can figure out the rebuild command on your own.
13+
#
14+
#
15+
# Commands:
16+
#
17+
# web: Stops currently-running delphi_web_epidata instances, if any.
18+
# Rebuilds delphi_web_epidata image.
19+
# Runs image in the background and pipes stdout to a log file.
20+
#
21+
# db: Stops currently-running delphi_database_epidata instances, if any.
22+
# Rebuilds delphi_database_epidata image.
23+
# Runs image in the background and pipes stdout to a log file.
24+
# Blocks until database is ready to receive connections.
25+
#
26+
# python: Rebuilds delphi_web_python image. You shouldn't need to do this
27+
# often; only if you are installing a new environment, or have
28+
# made changes to delphi-epidata/dev/docker/python/Dockerfile.
29+
#
30+
# all: Runs the commands 'web' 'db' and 'python'.
31+
#
32+
# test: Runs test and integrations in delphi-epidata. If test-subdir
33+
# optional arg is provided, then only the tests in that subdir
34+
# are run.
35+
#
36+
# clean: Cleans up dangling Docker images.
37+
#
38+
#
39+
# Optional arguments:
40+
# pdb=1 Drops you into debug mode upon test failure, if running tests.
41+
# test-subdir= Only runs tests in the directory provided here, e.g.
42+
# repos/delphi/delphi-epidata/tests/acquisition/covidcast
43+
44+
45+
# Set optional argument defaults
46+
ifdef pdb
47+
override pdb=--pdb
48+
else
49+
pdb=
50+
endif
51+
52+
ifdef test-subdir
53+
else
54+
test-subdir=repos/delphi/delphi-epidata/tests repos/delphi/delphi-epidata/integrations
55+
endif
56+
57+
# Set your shell here
58+
SHELL:=/bin/sh
59+
60+
# Get the Makefile's absolute path: https://stackoverflow.com/a/324782/4784655
61+
# (if called from symlink, the path is the location of the symlink)
62+
NOW:=$(shell date "+%Y-%m-%d")
63+
CWD:=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
64+
LOG_WEB:=$(CWD)docker-logs/delphi_web_epidata_$(NOW).log
65+
LOG_DB:=$(CWD)docker-logs/delphi_database_epidata_$(NOW).log
66+
67+
68+
.PHONY=web
69+
web:
70+
@# Stop container if running
71+
@-docker ps | grep delphi_web_epidata && docker stop delphi_web_epidata
72+
73+
@# Setup virtual network if it doesn't exist
74+
@-docker network ls | grep delphi-net || docker network create --driver bridge delphi-net
75+
76+
@# Build the web_epidata image
77+
@cd repos/delphi/delphi-epidata;\
78+
docker build -t delphi_web_epidata -f ./devops/Dockerfile .;\
79+
cd ../../../
80+
81+
@# Run the web server
82+
@docker run --rm -p 127.0.0.1:10080:80 \
83+
--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
84+
--env "FLASK_SECRET=abc" --env "FLASK_PREFIX=/epidata" \
85+
--network delphi-net --name delphi_web_epidata \
86+
delphi_web_epidata >$(LOG_WEB) 2>&1 &
87+
88+
.PHONY=db
89+
db:
90+
@# Stop container if running
91+
@-docker ps | grep delphi_database_epidata && docker stop delphi_database_epidata
92+
93+
@# Only build prereqs if we need them
94+
@-docker images delphi_database | grep delphi || \
95+
docker build -t delphi_database -f repos/delphi/operations/dev/docker/database/Dockerfile .
96+
97+
@# Build the database_epidata image
98+
@docker build -t delphi_database_epidata \
99+
-f repos/delphi/delphi-epidata/dev/docker/database/epidata/Dockerfile .
100+
101+
@# Run the database
102+
@docker run --rm -p 127.0.0.1:13306:3306 \
103+
--network delphi-net --name delphi_database_epidata \
104+
delphi_database_epidata >$(LOG_DB) 2>&1 &
105+
106+
@# Block until DB is ready
107+
@while true; do \
108+
sed -n '/Temporary server stopped/,/mysqld: ready for connections/p' $(LOG_DB) | grep "ready for connections" && break; \
109+
tail -1 $(LOG_DB); \
110+
sleep 1; \
111+
done
112+
113+
.PHONY=py
114+
py:
115+
@# Build the python image
116+
@docker build -t delphi_python \
117+
-f repos/delphi/operations/dev/docker/python/Dockerfile .
118+
119+
@docker build -t delphi_web_python \
120+
-f repos/delphi/delphi-epidata/dev/docker/python/Dockerfile .
121+
122+
.PHONY=all
123+
all: web db py
124+
125+
.PHONY=test
126+
test:
127+
@docker run -i --rm --network delphi-net \
128+
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata,target=/usr/src/app/repos/delphi/delphi-epidata,readonly \
129+
--mount type=bind,source=$(CWD)repos/delphi/delphi-epidata/src,target=/usr/src/app/delphi/epidata,readonly \
130+
--env "SQLALCHEMY_DATABASE_URI=mysql+mysqldb://user:pass@delphi_database_epidata:3306/epidata" \
131+
--env "FLASK_SECRET=abc" \
132+
delphi_web_python python -m pytest --import-mode importlib $(pdb) $(test-subdir) | tee -a output.txt
133+
134+
.PHONY=clean
135+
clean:
136+
docker images -f "dangling=true" -q | xargs docker rmi >/dev/null 2>&1

‎dev/local/epidata-refresh.sh

-104
This file was deleted.

‎dev/local/install.sh

+20-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,26 @@
11
#!/bin/bash
2+
# Bootstrap delphi-epidata
3+
#
4+
# Downloads the repos needed for local delphi-epidata development into current dir
5+
# and provides a Makefile with Docker control commands.
26

3-
mkdir -p driver/repos/delphi driver-logs/delphi_database_epidata driver-logs/delphi_web_epidata
4-
cd driver/repos/delphi
7+
mkdir -p repos/delphi
8+
cd repos/delphi
59
git clone https://github.com/cmu-delphi/operations
610
git clone https://github.com/cmu-delphi/delphi-epidata
711
git clone https://github.com/cmu-delphi/utils
12+
git clone https://github.com/cmu-delphi/flu-contest
13+
git clone https://github.com/cmu-delphi/nowcast
14+
git clone https://github.com/cmu-delphi/github-deploy-repo
815
cd ../../
9-
ln -s repos/delphi/delphi-epidata/dev/local/epidata-refresh.sh
16+
17+
mkdir -p repos/undefx
18+
cd repos/undefx
19+
git clone https://github.com/undefx/py3tester
20+
git clone https://github.com/undefx/undef-analysis
21+
cd ../../
22+
23+
mkdir -p docker-logs
24+
25+
ln -s repos/delphi/delphi-epidata/dev/local/Makefile
26+
ln -s repos/delphi/delphi-epidata/dev/local/.dockerignore

‎docs/epidata_development.md

+8-2
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,16 @@ nav_order: 4
77

88
## Quickstart
99

10+
In the directory where you want to work run the following
11+
1012
```
1113
$ curl "https://raw.githubusercontent.com/cmu-delphi/delphi-epidata/dev/dev/local/install.sh" | bash
12-
$ cd driver
13-
$ [sudo] ./epidata-refresh.sh database web python
14+
$ [sudo] make all
15+
$ make test
16+
# To drop into debugger on error
17+
$ make test pdb=1
18+
# To test only a subset of tests
19+
$ make test test-subdir=repos/delphi/delphi-epidata/integrations/acquisition
1420
```
1521
(sudo requirement depends on your Docker installation and operating system)
1622

0 commit comments

Comments
 (0)