Skip to content

Commit 18c78c1

Browse files
daniel-goldsteindaniel-goldstein
authored
[docker] Put Makefile image-building in the top-level Makefile (#12841)
Got annoyed with the constant re-tagging of images that don't need to be rebuilt, and decided to play a little make golf along the way. cc @jigold This should dramatically reduce the number of tags for hail-ubuntu from make-deployed images, though the number of layers in the container registry should not change.
1 parent a9fea66 commit 18c78c1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

56 files changed

+275
-444
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,12 @@ hail/upload-qob-jar
3232
hail/upload-qob-test-resources
3333
**/hail_version
3434
web_common/web_common/static/css/
35-
website/docs.tar.gz
35+
docs.tar.gz
3636
website/website/static/css/
3737
*.dylib # macOS dynamic libraries
3838
*/hail.jar
3939
infra/.terraform.lock.hcl
4040
.dist/
4141
.cache/
4242
wheel-container.tar
43+
*-image

‎Makefile

+101-37
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,27 @@
11
.DEFAULT_GOAL := default
22

3+
include config.mk
4+
5+
SERVICES := auth batch ci memory notebook monitoring website
6+
SERVICES_IMAGES := $(patsubst %, %-image, $(SERVICES))
7+
SERVICES_MODULES := $(SERVICES) gear web_common
8+
CHECK_SERVICES_MODULES := $(patsubst %, check-%, $(SERVICES_MODULES))
9+
10+
HAILTOP_VERSION := hail/python/hailtop/hail_version
11+
SERVICES_IMAGE_DEPS = hail-ubuntu-image $(HAILTOP_VERSION) $(shell git ls-files hail/python/hailtop gear web_common)
12+
13+
EMPTY :=
14+
SPACE := $(EMPTY) $(EMPTY)
15+
EXTRA_PYTHONPATH := hail/python:$(subst $(SPACE),:,$(SERVICES_MODULES))
16+
17+
PYTHONPATH ?= ""
18+
ifeq ($(PYTHONPATH), "")
19+
PYTHONPATH := $(EXTRA_PYTHONPATH)
20+
else
21+
PYTHONPATH := $(PYTHONPATH):$(EXTRA_PYTHONPATH)
22+
endif
23+
PYTHON := PYTHONPATH="$(PYTHONPATH)" python3
24+
325
default:
426
@echo Do not use this makefile to build hail, for information on how to \
527
build hail see: https://hail.is/docs/0.2/
@@ -13,43 +35,17 @@ check-hail:
1335
$(MAKE) -C hail/python check
1436

1537
.PHONY: check-services
16-
check-services: check-auth check-batch check-ci check-gear check-memory \
17-
check-notebook check-monitoring check-web-common check-website
18-
19-
.PHONY: check-auth
20-
check-auth:
21-
$(MAKE) -C auth check
22-
23-
.PHONY: check-batch
24-
check-batch:
25-
$(MAKE) -C batch check
26-
27-
.PHONY: check-ci
28-
check-ci:
29-
$(MAKE) -C ci check
30-
31-
.PHONY: check-gear
32-
check-gear:
33-
$(MAKE) -C gear check
34-
35-
.PHONY: check-memory
36-
check-memory:
37-
$(MAKE) -C memory check
38-
39-
.PHONY: check-notebook
40-
check-notebook:
41-
$(MAKE) -C notebook check
42-
43-
.PHONY: check-monitoring
44-
$(MAKE) -C monitoring check
45-
46-
.PHONY: check-web-common
47-
check-web-common:
48-
$(MAKE) -C web_common check
49-
50-
.PHONY: check-website
51-
check-website:
52-
$(MAKE) -C website check
38+
check-services: $(CHECK_SERVICES_MODULES)
39+
40+
.PHONY: check-%
41+
$(CHECK_SERVICES_MODULES): check-%:
42+
$(PYTHON) -m flake8 --config setup.cfg $*
43+
$(PYTHON) -m pylint --rcfile pylintrc --recursive=y $* --score=n
44+
$(PYTHON) -m mypy --config-file setup.cfg $*
45+
$(PYTHON) -m isort $* --check-only --diff
46+
$(PYTHON) -m black $* --line-length=120 --skip-string-normalization --check --diff
47+
curlylint $*
48+
cd $* && bash ../check-sql.sh
5349

5450
.PHONY: check-pip-requirements
5551
check-pip-requirements:
@@ -113,3 +109,71 @@ generate-pip-lockfiles: auth/pinned-requirements.txt
113109
generate-pip-lockfiles: batch/pinned-requirements.txt
114110
generate-pip-lockfiles: ci/pinned-requirements.txt
115111
generate-pip-lockfiles: memory/pinned-requirements.txt
112+
113+
$(HAILTOP_VERSION):
114+
$(MAKE) -C hail python/hailtop/hail_version
115+
116+
hail-ubuntu-image: $(shell git ls-files docker/hail-ubuntu)
117+
$(eval HAIL_UBUNTU_IMAGE := $(DOCKER_PREFIX)/hail-ubuntu:$(TOKEN))
118+
python3 ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' docker/hail-ubuntu/Dockerfile docker/hail-ubuntu/Dockerfile.out
119+
./docker-build.sh docker/hail-ubuntu Dockerfile.out $(HAIL_UBUNTU_IMAGE)
120+
echo $(HAIL_UBUNTU_IMAGE) > $@
121+
122+
base-image: hail-ubuntu-image docker/Dockerfile.base
123+
$(eval BASE_IMAGE := $(DOCKER_PREFIX)/base:$(TOKEN))
124+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' docker/Dockerfile.base docker/Dockerfile.base.out
125+
./docker-build.sh . docker/Dockerfile.base.out $(BASE_IMAGE)
126+
echo $(BASE_IMAGE) > $@
127+
128+
private-repo-hailgenetics-hail-image: hail-ubuntu-image docker/hailgenetics/hail/Dockerfile $(shell git ls-files hail/src/main hail/python)
129+
$(eval PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE := $(DOCKER_PREFIX)/hailgenetics/hail:$(TOKEN))
130+
$(MAKE) -C hail wheel
131+
tar -cvf wheel-container.tar \
132+
-C hail/build/deploy/dist \
133+
hail-$$(cat hail/python/hail/hail_pip_version)-py3-none-any.whl
134+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' docker/hailgenetics/hail/Dockerfile docker/hailgenetics/hail/Dockerfile.out
135+
./docker-build.sh . docker/hailgenetics/hail/Dockerfile.out $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE)
136+
rm wheel-container.tar
137+
echo $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE) > $@
138+
139+
.PHONY: docs
140+
docs:
141+
$(MAKE) -C hail hail-docs-no-test batch-docs
142+
gcloud storage cp gs://hail-common/builds/0.1/docs/hail-0.1-docs-5a6778710097.tar.gz .
143+
mkdir -p hail/build/www/docs/0.1
144+
tar -xvf hail-0.1-docs-5a6778710097.tar.gz -C hail/build/www/docs/0.1 --strip-components 2
145+
rm hail-0.1-docs-5a6778710097.tar.gz
146+
tar czf docs.tar.gz -C hail/build/www .
147+
148+
website-image: docs
149+
150+
$(SERVICES_IMAGES): %-image: $(SERVICES_IMAGE_DEPS) $(shell git ls-files $$*)
151+
$(eval IMAGE := $(DOCKER_PREFIX)/$*:$(TOKEN))
152+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' $*/Dockerfile $*/Dockerfile.out
153+
./docker-build.sh . $*/Dockerfile.out $(IMAGE)
154+
echo $(IMAGE) > $@
155+
156+
ci-utils-image: hail-ubuntu-image $(SERVICES_IMAGE_DEPS) $(shell git ls-files ci)
157+
$(eval CI_UTILS_IMAGE := $(DOCKER_PREFIX)/ci-utils:$(TOKEN))
158+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' ci/Dockerfile.ci-utils ci/Dockerfile.ci-utils.out
159+
./docker-build.sh . ci/Dockerfile.ci-utils.out $(CI_UTILS_IMAGE)
160+
echo $(CI_UTILS_IMAGE) > $@
161+
162+
hail-buildkit-image: ci/buildkit/Dockerfile
163+
$(eval HAIL_BUILDKIT_IMAGE := $(DOCKER_PREFIX)/hail-buildkit:$(TOKEN))
164+
python3 ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' ci/buildkit/Dockerfile ci/buildkit/Dockerfile.out
165+
./docker-build.sh ci buildkit/Dockerfile.out $(HAIL_BUILDKIT_IMAGE)
166+
echo $(HAIL_BUILDKIT_IMAGE) > $@
167+
168+
batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar:
169+
mkdir -p batch/jars
170+
cd batch/jars && curl -LO https://github.com/kohlschutter/junixsocket/releases/download/junixsocket-parent-2.3.3/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
171+
172+
batch/src/main/java/is/hail/JVMEntryway.class: batch/src/main/java/is/hail/JVMEntryway.java batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
173+
javac -cp batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar $<
174+
175+
batch-worker-image: batch/src/main/java/is/hail/JVMEntryway.class $(SERVICES_IMAGE_DEPS) $(shell git ls-files batch)
176+
$(eval BATCH_WORKER_IMAGE := $(DOCKER_PREFIX)/batch-worker:$(TOKEN))
177+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"},"global":{"cloud":"$(CLOUD)"}}' batch/Dockerfile.worker batch/Dockerfile.worker.out
178+
./docker-build.sh . batch/Dockerfile.worker.out $(BATCH_WORKER_IMAGE)
179+
echo $(BATCH_WORKER_IMAGE) > $@

‎auth/Dockerfile

-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ RUN hail-pip-install \
1313
COPY hail/python/setup-hailtop.py /hailtop/setup.py
1414
COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in
1515
COPY hail/python/hailtop /hailtop/hailtop/
16-
COPY /hail_version /hailtop/hailtop/hail_version
1716

1817
COPY gear/setup.py /gear/setup.py
1918
COPY gear/gear /gear/gear/

‎auth/Makefile

+2-19
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,14 @@
11
include ../config.mk
22

3-
AUTH_IMAGE := $(DOCKER_PREFIX)/auth:$(TOKEN)
4-
5-
EXTRA_PYTHONPATH := ../hail/python:../gear:../web_common
6-
PYTHON := PYTHONPATH=$${PYTHONPATH:+$${PYTHONPATH}:}$(EXTRA_PYTHONPATH) python3
7-
BLACK := $(PYTHON) -m black . --line-length=120 --skip-string-normalization
8-
9-
.PHONY: check
10-
check:
11-
$(PYTHON) -m flake8 auth
12-
$(PYTHON) -m pylint --rcfile ../pylintrc auth --score=n
13-
$(PYTHON) -m isort . --check-only --diff
14-
$(BLACK) --check --diff
15-
curlylint .
16-
bash ../check-sql.sh
17-
183
.PHONY: build
194
build:
20-
$(MAKE) -C ../docker hail-ubuntu
21-
python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat ../docker/hail-ubuntu-image-ref)'"}}' Dockerfile Dockerfile.out
22-
../docker-build.sh .. auth/Dockerfile.out $(AUTH_IMAGE)
5+
$(MAKE) -C .. auth-image
236

247
.PHONY: deploy
258
deploy: build
269
! [ -z $(NAMESPACE) ] # call this like: make deploy NAMESPACE=default
2710
kubectl -n $(NAMESPACE) apply -f auth-driver-service-account.yaml
28-
python3 ../ci/jinja2_render.py '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"default_ns":{"name":"$(NAMESPACE)"},"auth_image":{"image":"$(AUTH_IMAGE)"},"auth_database":{"user_secret_name":"sql-auth-user-config"}}' deployment.yaml deployment.yaml.out
11+
python3 ../ci/jinja2_render.py '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"scope":"$(SCOPE)","default_ns":{"name":"$(NAMESPACE)"},"auth_image":{"image":"'$$(cat ../auth-image)'"},"auth_database":{"user_secret_name":"sql-auth-user-config"}}' deployment.yaml deployment.yaml.out
2912
kubectl -n $(NAMESPACE) apply -f deployment.yaml.out
3013

3114
.PHONY:

‎auth/auth/driver/driver.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import os
66
import random
77
import secrets
8-
from typing import Optional
8+
from typing import Any, Awaitable, Callable, Dict, List, Optional
99

1010
import aiohttp
1111
import kubernetes_asyncio.client
@@ -521,7 +521,7 @@ async def _create_user(app, user, skip_trial_bp, cleanup):
521521

522522

523523
async def create_user(app, user, skip_trial_bp=False):
524-
cleanup = []
524+
cleanup: List[Callable[[], Awaitable[None]]] = []
525525
try:
526526
await _create_user(app, user, skip_trial_bp, cleanup)
527527
except Exception:
@@ -610,7 +610,7 @@ async def update_users(app):
610610

611611

612612
async def async_main():
613-
app = {}
613+
app: Dict[str, Any] = {}
614614

615615
user_creation_loop = None
616616
try:
@@ -629,7 +629,7 @@ async def async_main():
629629

630630
app['identity_client'] = get_identity_client()
631631

632-
app['batch_client'] = await bc.aioclient.BatchClient.create(None)
632+
app['batch_client'] = await bc.aioclient.BatchClient.create('')
633633

634634
users_changed_event = asyncio.Event()
635635
app['users_changed_event'] = users_changed_event

‎auth/auth/flow.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ def initiate_flow(self, redirect_uri: str) -> dict:
9292
}
9393

9494
def receive_callback(self, request: aiohttp.web.Request, flow_dict: dict) -> FlowResult:
95-
query_dict = urllib.parse.parse_qs(request.query_string)
96-
query_dict = {k: v[0] for k, v in query_dict.items()}
95+
query_key_to_list_of_values = urllib.parse.parse_qs(request.query_string)
96+
query_dict = {k: v[0] for k, v in query_key_to_list_of_values.items()}
9797

9898
token = self._client.acquire_token_by_auth_code_flow(flow_dict['flow'], query_dict)
9999

‎batch/Dockerfile

-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ RUN hail-pip-install \
1313
COPY hail/python/setup-hailtop.py /hailtop/setup.py
1414
COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in
1515
COPY hail/python/hailtop /hailtop/hailtop/
16-
COPY /hail_version /hailtop/hailtop/hail_version
1716

1817
COPY gear/setup.py /gear/setup.py
1918
COPY gear/gear /gear/gear/

‎batch/Dockerfile.worker

-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ RUN git clone --depth 1 --branch 1.4.4 https://github.com/containers/crun.git &&
6060
FROM base
6161
COPY --from=crun_builder /usr/local/bin/crun /usr/local/bin/crun
6262
COPY hail/python/setup-hailtop.py /hailtop/setup.py
63-
COPY /hail_version /hailtop/hailtop/hail_version
6463
COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in
6564
COPY hail/python/hailtop /hailtop/hailtop/
6665

‎batch/Makefile

+3-44
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,10 @@
11
include ../config.mk
22

3-
BATCH_IMAGE := $(DOCKER_PREFIX)/batch:$(TOKEN)
4-
BATCH_WORKER_IMAGE := $(DOCKER_PREFIX)/batch-worker:$(TOKEN)
5-
6-
EXTRA_PYTHONPATH := ../hail/python:../gear:../web_common
7-
PYTHON := PYTHONPATH=$${PYTHONPATH:+$${PYTHONPATH}:}$(EXTRA_PYTHONPATH) python3
8-
CLOUD := $(shell kubectl get secret global-config --template={{.data.cloud}} | base64 --decode)
9-
10-
BLACK := $(PYTHON) -m black . --line-length=120 --skip-string-normalization
11-
12-
.PHONY: check
13-
check:
14-
$(PYTHON) -m flake8 --config ../setup.cfg batch
15-
$(PYTHON) -m pylint --rcfile ../pylintrc batch --score=n
16-
$(PYTHON) -m mypy --config-file ../setup.cfg batch
17-
$(PYTHON) -m isort . --check-only --diff
18-
$(BLACK) --check --diff
19-
curlylint .
20-
bash ../check-sql.sh
21-
22-
.PHONY: hail-ubuntu
23-
hail-ubuntu:
24-
$(MAKE) -C ../docker hail-ubuntu
25-
26-
.PHONY: build-batch
27-
build-batch: hail-ubuntu
28-
python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat ../docker/hail-ubuntu-image-ref)'"}}' Dockerfile Dockerfile.out
29-
../docker-build.sh . Dockerfile.out $(BATCH_IMAGE)
30-
31-
jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar:
32-
mkdir jars
33-
cd jars && curl -LO https://github.com/kohlschutter/junixsocket/releases/download/junixsocket-parent-2.3.3/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
34-
35-
src/main/java/is/hail/JVMEntryway.class: src/main/java/is/hail/JVMEntryway.java jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
36-
javac -cp jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar $<
37-
38-
.PHONY: build-worker
39-
build-worker: src/main/java/is/hail/JVMEntryway.class hail-ubuntu
40-
$(MAKE) -C ../hail python/hail/hail_version
41-
cp ../hail/python/hail/hail_version ../hail_version
42-
python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat ../docker/hail-ubuntu-image-ref)'"},"global":{"cloud":"$(CLOUD)"}}' Dockerfile.worker Dockerfile.worker.out
43-
../docker-build.sh .. batch/Dockerfile.worker.out $(BATCH_WORKER_IMAGE)
44-
453
.PHONY: build
46-
build: build-batch build-worker
4+
build:
5+
$(MAKE) -C .. batch-image batch-worker-image
476

48-
JINJA_ENVIRONMENT = '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"batch_image":{"image":"$(BATCH_IMAGE)"},"batch_worker_image":{"image":"$(BATCH_WORKER_IMAGE)"},"default_ns":{"name":"$(NAMESPACE)"},"batch_database":{"user_secret_name":"sql-batch-user-config"},"scope":"$(SCOPE)","global":{"docker_prefix":"$(DOCKER_PREFIX)"}}'
7+
JINJA_ENVIRONMENT = '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"batch_image":{"image":"$(shell cat ../batch-image)"},"batch_worker_image":{"image":"$(shell cat ../batch-worker-image)"},"default_ns":{"name":"$(NAMESPACE)"},"batch_database":{"user_secret_name":"sql-batch-user-config"},"scope":"$(SCOPE)","global":{"docker_prefix":"$(DOCKER_PREFIX)"}}'
498

509
.PHONY: deploy
5110
deploy: build

‎batch/test/billing_projects.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,14 @@ def get_billing_project_prefix():
1010

1111
async def delete_all_test_billing_projects():
1212
billing_project_prefix = get_billing_project_prefix()
13-
bc = await BatchClient.create(None, token_file=os.environ['HAIL_TEST_DEV_TOKEN_FILE'])
13+
bc = await BatchClient.create('', token_file=os.environ['HAIL_TEST_DEV_TOKEN_FILE'])
1414
try:
1515
for project in await bc.list_billing_projects():
1616
if project['billing_project'].startswith(billing_project_prefix):
1717
try:
1818
print(f'deleting {project}')
1919
await bc.delete_billing_project(project['billing_project'])
20-
except Exception as exc:
20+
except Exception:
2121
print(f'exception deleting {project}; will continue')
2222
traceback.print_exc()
2323
finally:

‎batch/test/conftest.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,12 @@ def log_before_after():
1414
log.info('ending test')
1515

1616

17-
def pytest_collection_modifyitems(config, items):
17+
def pytest_collection_modifyitems(config, items): # pylint: disable=unused-argument
1818
n_splits = int(os.environ.get('HAIL_RUN_IMAGE_SPLITS', '1'))
1919
split_index = int(os.environ.get('HAIL_RUN_IMAGE_SPLIT_INDEX', '-1'))
2020
if n_splits <= 1:
2121
return
22-
if not (0 <= split_index < n_splits):
22+
if not 0 <= split_index < n_splits:
2323
raise RuntimeError(f"invalid split_index: index={split_index}, n_splits={n_splits}\n env={os.environ}")
2424
skip_this = pytest.mark.skip(reason="skipped in this round")
2525

‎batch/test/failure_injecting_client_session.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
class FailureInjectingClientSession(httpx.ClientSession):
88
def __init__(self, should_fail):
9+
super().__init__()
910
self.should_fail = should_fail
1011
self.real_session = httpx.client_session()
1112

@@ -24,6 +25,6 @@ def maybe_fail(self, method, path, headers):
2425
history=(),
2526
)
2627

27-
async def request(self, method, path, *args, **kwargs):
28+
def request(self, method, path, *args, **kwargs):
2829
self.maybe_fail(method, path, kwargs.get('headers', {}))
29-
return await self.real_session.request(method, path, *args, **kwargs)
30+
return self.real_session.request(method, path, *args, **kwargs)

0 commit comments

Comments
 (0)