Skip to content

Commit 49c179c

Browse files
illusionalillusional
committed
Merge commit 'cf32652c50771e7f81b1bfa618f6e3f399cd238d' into upstream-113
# Conflicts: # .gitignore # auth/Makefile # batch/batch/driver/instance_collection/pool.py # batch/batch/driver/main.py # build.yaml # ci/create_initial_account.py # docker/Dockerfile.service-base # hail/src/main/scala/is/hail/backend/service/ServiceBackend.scala # hail/src/main/scala/is/hail/backend/service/Worker.scala
2 parents 39116b8 + cf32652 commit 49c179c

File tree

349 files changed

+9489
-6221
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

349 files changed

+9489
-6221
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,13 @@ hail/upload-qob-jar
3535
hail/upload-qob-test-resources
3636
**/hail_version
3737
web_common/web_common/static/css/
38-
website/docs.tar.gz
38+
docs.tar.gz
3939
website/website/static/css/
4040
*.dylib # macOS dynamic libraries
4141
*/hail.jar
4242
infra/.terraform.lock.hcl
4343
.dist/
4444
.cache/
4545
wheel-container.tar
46+
*-image
4647
_/

‎Makefile

+154-33
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,27 @@
11
.DEFAULT_GOAL := default
22

3+
include config.mk
4+
5+
SERVICES := auth batch ci memory notebook monitoring website
6+
SERVICES_IMAGES := $(patsubst %, %-image, $(SERVICES))
7+
SERVICES_MODULES := $(SERVICES) gear web_common
8+
CHECK_SERVICES_MODULES := $(patsubst %, check-%, $(SERVICES_MODULES))
9+
10+
HAILTOP_VERSION := hail/python/hailtop/hail_version
11+
SERVICES_IMAGE_DEPS = hail-ubuntu-image $(HAILTOP_VERSION) $(shell git ls-files hail/python/hailtop gear web_common)
12+
13+
EMPTY :=
14+
SPACE := $(EMPTY) $(EMPTY)
15+
EXTRA_PYTHONPATH := hail/python:$(subst $(SPACE),:,$(SERVICES_MODULES))
16+
17+
PYTHONPATH ?= ""
18+
ifeq ($(PYTHONPATH), "")
19+
PYTHONPATH := $(EXTRA_PYTHONPATH)
20+
else
21+
PYTHONPATH := $(PYTHONPATH):$(EXTRA_PYTHONPATH)
22+
endif
23+
PYTHON := PYTHONPATH="$(PYTHONPATH)" python3
24+
325
default:
426
@echo Do not use this makefile to build hail, for information on how to \
527
build hail see: https://hail.is/docs/0.2/
@@ -13,46 +35,145 @@ check-hail:
1335
$(MAKE) -C hail/python check
1436

1537
.PHONY: check-services
16-
check-services: check-auth check-batch check-ci check-gear check-memory \
17-
check-notebook check-monitoring check-web-common check-website
38+
check-services: $(CHECK_SERVICES_MODULES)
39+
40+
.PHONY: check-%
41+
$(CHECK_SERVICES_MODULES): check-%:
42+
$(PYTHON) -m flake8 --config setup.cfg $*
43+
$(PYTHON) -m pylint --rcfile pylintrc --recursive=y $* --score=n
44+
$(PYTHON) -m mypy --config-file setup.cfg $*
45+
$(PYTHON) -m isort $* --check-only --diff
46+
$(PYTHON) -m black $* --line-length=120 --skip-string-normalization --check --diff
47+
curlylint $*
48+
cd $* && bash ../check-sql.sh
49+
50+
.PHONY: check-pip-requirements
51+
check-pip-requirements:
52+
./check_pip_requirements.sh \
53+
hail/python/hailtop \
54+
hail/python \
55+
hail/python/dev \
56+
gear \
57+
web_common \
58+
auth \
59+
batch \
60+
ci \
61+
memory
62+
63+
.PHONY: install-dev-requirements
64+
install-dev-requirements:
65+
python3 -m pip install \
66+
-r hail/python/pinned-requirements.txt \
67+
-r hail/python/dev/pinned-requirements.txt \
68+
-r gear/pinned-requirements.txt \
69+
-r web_common/pinned-requirements.txt \
70+
-r auth/pinned-requirements.txt \
71+
-r batch/pinned-requirements.txt \
72+
-r ci/pinned-requirements.txt \
73+
-r memory/pinned-requirements.txt \
74+
75+
hail/python/hailtop/pinned-requirements.txt: hail/python/hailtop/requirements.txt
76+
./generate-linux-pip-lockfile.sh hail/python/hailtop
77+
78+
hail/python/pinned-requirements.txt: hail/python/requirements.txt hail/python/hailtop/pinned-requirements.txt
79+
./generate-linux-pip-lockfile.sh hail/python
80+
81+
hail/python/dev/pinned-requirements.txt: hail/python/dev/requirements.txt hail/python/pinned-requirements.txt
82+
./generate-linux-pip-lockfile.sh hail/python/dev
83+
84+
gear/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
85+
./generate-linux-pip-lockfile.sh gear
86+
87+
web_common/pinned-requirements.txt: gear/pinned-requirements.txt web_common/requirements.txt
88+
./generate-linux-pip-lockfile.sh web_common
89+
90+
auth/pinned-requirements.txt: web_common/pinned-requirements.txt auth/requirements.txt
91+
./generate-linux-pip-lockfile.sh auth
92+
93+
batch/pinned-requirements.txt: web_common/pinned-requirements.txt batch/requirements.txt
94+
./generate-linux-pip-lockfile.sh batch
95+
96+
ci/pinned-requirements.txt: web_common/pinned-requirements.txt ci/requirements.txt
97+
./generate-linux-pip-lockfile.sh ci
98+
99+
memory/pinned-requirements.txt: gear/pinned-requirements.txt memory/requirements.txt
100+
./generate-linux-pip-lockfile.sh memory
101+
102+
.PHONY: generate-pip-lockfiles
103+
generate-pip-lockfiles: hail/python/hailtop/pinned-requirements.txt
104+
generate-pip-lockfiles: hail/python/pinned-requirements.txt
105+
generate-pip-lockfiles: hail/python/dev/pinned-requirements.txt
106+
generate-pip-lockfiles: gear/pinned-requirements.txt
107+
generate-pip-lockfiles: web_common/pinned-requirements.txt
108+
generate-pip-lockfiles: auth/pinned-requirements.txt
109+
generate-pip-lockfiles: batch/pinned-requirements.txt
110+
generate-pip-lockfiles: ci/pinned-requirements.txt
111+
generate-pip-lockfiles: memory/pinned-requirements.txt
112+
113+
$(HAILTOP_VERSION):
114+
$(MAKE) -C hail python/hailtop/hail_version
115+
116+
hail-ubuntu-image: $(shell git ls-files docker/hail-ubuntu)
117+
$(eval HAIL_UBUNTU_IMAGE := $(DOCKER_PREFIX)/hail-ubuntu:$(TOKEN))
118+
python3 ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' docker/hail-ubuntu/Dockerfile docker/hail-ubuntu/Dockerfile.out
119+
./docker-build.sh docker/hail-ubuntu Dockerfile.out $(HAIL_UBUNTU_IMAGE)
120+
echo $(HAIL_UBUNTU_IMAGE) > $@
18121

19-
.PHONY: check-auth
20-
check-auth:
21-
$(MAKE) -C auth check
122+
base-image: hail-ubuntu-image docker/Dockerfile.base
123+
$(eval BASE_IMAGE := $(DOCKER_PREFIX)/base:$(TOKEN))
124+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' docker/Dockerfile.base docker/Dockerfile.base.out
125+
./docker-build.sh . docker/Dockerfile.base.out $(BASE_IMAGE)
126+
echo $(BASE_IMAGE) > $@
22127

23-
.PHONY: check-batch
24-
check-batch:
25-
$(MAKE) -C batch check
128+
private-repo-hailgenetics-hail-image: hail-ubuntu-image docker/hailgenetics/hail/Dockerfile $(shell git ls-files hail/src/main hail/python)
129+
$(eval PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE := $(DOCKER_PREFIX)/hailgenetics/hail:$(TOKEN))
130+
$(MAKE) -C hail wheel
131+
tar -cvf wheel-container.tar \
132+
-C hail/build/deploy/dist \
133+
hail-$$(cat hail/python/hail/hail_pip_version)-py3-none-any.whl
134+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' docker/hailgenetics/hail/Dockerfile docker/hailgenetics/hail/Dockerfile.out
135+
./docker-build.sh . docker/hailgenetics/hail/Dockerfile.out $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE)
136+
rm wheel-container.tar
137+
echo $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE) > $@
26138

27-
.PHONY: check-ci
28-
check-ci:
29-
$(MAKE) -C ci check
139+
.PHONY: docs
140+
docs:
141+
$(MAKE) -C hail hail-docs-no-test batch-docs
142+
gcloud storage cp gs://hail-common/builds/0.1/docs/hail-0.1-docs-5a6778710097.tar.gz .
143+
mkdir -p hail/build/www/docs/0.1
144+
tar -xvf hail-0.1-docs-5a6778710097.tar.gz -C hail/build/www/docs/0.1 --strip-components 2
145+
rm hail-0.1-docs-5a6778710097.tar.gz
146+
tar czf docs.tar.gz -C hail/build/www .
30147

31-
.PHONY: check-gear
32-
check-gear:
33-
$(MAKE) -C gear check
148+
website-image: docs
34149

35-
.PHONY: check-memory
36-
check-memory:
37-
$(MAKE) -C memory check
150+
$(SERVICES_IMAGES): %-image: $(SERVICES_IMAGE_DEPS) $(shell git ls-files $$*)
151+
$(eval IMAGE := $(DOCKER_PREFIX)/$*:$(TOKEN))
152+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' $*/Dockerfile $*/Dockerfile.out
153+
./docker-build.sh . $*/Dockerfile.out $(IMAGE)
154+
echo $(IMAGE) > $@
38155

39-
.PHONY: check-notebook
40-
check-notebook:
41-
$(MAKE) -C notebook check
156+
ci-utils-image: base-image $(SERVICES_IMAGE_DEPS) $(shell git ls-files ci)
157+
$(eval CI_UTILS_IMAGE := $(DOCKER_PREFIX)/ci-utils:$(TOKEN))
158+
python3 ci/jinja2_render.py '{"base_image":{"image":"'$$(cat base-image)'"}}' ci/Dockerfile.ci-utils ci/Dockerfile.ci-utils.out
159+
./docker-build.sh . ci/Dockerfile.ci-utils.out $(CI_UTILS_IMAGE)
160+
echo $(CI_UTILS_IMAGE) > $@
42161

43-
.PHONY: check-monitoring
44-
$(MAKE) -C monitoring check
162+
hail-buildkit-image: ci/buildkit/Dockerfile
163+
$(eval HAIL_BUILDKIT_IMAGE := $(DOCKER_PREFIX)/hail-buildkit:$(TOKEN))
164+
python3 ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' ci/buildkit/Dockerfile ci/buildkit/Dockerfile.out
165+
./docker-build.sh ci buildkit/Dockerfile.out $(HAIL_BUILDKIT_IMAGE)
166+
echo $(HAIL_BUILDKIT_IMAGE) > $@
45167

46-
.PHONY: check-web-common
47-
check-web-common:
48-
$(MAKE) -C web_common check
168+
batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar:
169+
mkdir -p batch/jars
170+
cd batch/jars && curl -LO https://github.com/kohlschutter/junixsocket/releases/download/junixsocket-parent-2.3.3/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
49171

50-
.PHONY: check-website
51-
check-website:
52-
$(MAKE) -C website check
172+
batch/src/main/java/is/hail/JVMEntryway.class: batch/src/main/java/is/hail/JVMEntryway.java batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
173+
javac -cp batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar $<
53174

54-
.PHONY: check-pip-dependencies
55-
check-pip-dependencies:
56-
./check_pip_requirements.sh hail/python/requirements.txt hail/python/pinned-requirements.txt
57-
./check_pip_requirements.sh hail/python/dev/requirements.txt hail/python/dev/pinned-requirements.txt
58-
./check_pip_requirements.sh docker/requirements.txt docker/linux-pinned-requirements.txt
175+
batch-worker-image: batch/src/main/java/is/hail/JVMEntryway.class $(SERVICES_IMAGE_DEPS) $(shell git ls-files batch)
176+
$(eval BATCH_WORKER_IMAGE := $(DOCKER_PREFIX)/batch-worker:$(TOKEN))
177+
python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"},"global":{"cloud":"$(CLOUD)"}}' batch/Dockerfile.worker batch/Dockerfile.worker.out
178+
./docker-build.sh . batch/Dockerfile.worker.out $(BATCH_WORKER_IMAGE)
179+
echo $(BATCH_WORKER_IMAGE) > $@

‎admin-pod/Dockerfile

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FROM {{ hail_ubuntu_image.image }}
2+
3+
# https://bugs.mysql.com/bug.php?id=105288&thanks=sub
4+
RUN hail-apt-get-install xz-utils libncurses5 && \
5+
curl --remote-name https://downloads.mysql.com/archives/get/p/23/file/mysql-8.0.26-linux-glibc2.17-x86_64-minimal-rebuild.tar.xz && \
6+
mkdir -p /opt && \
7+
tar -vx -C /opt -f mysql-8.0.26-linux-glibc2.17-x86_64-minimal-rebuild.tar.xz && \
8+
ln -s /opt/mysql-8.0.26-linux-glibc2.17-x86_64-minimal-rebuild/bin/* /usr/bin/

‎admin-pod/Makefile

+12-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,19 @@
11
include ../config.mk
22

3+
ADMIN_POD_IMAGE := $(DOCKER_PREFIX)/admin-pod:$(TOKEN)
4+
5+
.PHONY: hail-ubuntu
6+
hail-ubuntu:
7+
$(MAKE) -C ../docker hail-ubuntu
8+
9+
.PHONY: build
10+
build: hail-ubuntu
11+
python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat ../docker/hail-ubuntu-image-ref)'"}}' Dockerfile Dockerfile.out
12+
../docker-build.sh . Dockerfile.out $(ADMIN_POD_IMAGE)
13+
314
.PHONY: deploy
415
deploy:
516
! [ -z $(NAMESPACE) ] # call this like: make deploy NAMESPACE=default
617
$(MAKE) -C ../docker service-base
7-
python3 ../ci/jinja2_render.py '{"deploy":$(DEPLOY),"service_base_image":{"image":"'$$(cat ../docker/service-base-image-ref)'"}}' admin-pod.yaml admin-pod.yaml.out
18+
python3 ../ci/jinja2_render.py '{"deploy":$(DEPLOY),"admin_pod_image":{"image":"$(ADMIN_POD_IMAGE)"}}' admin-pod.yaml admin-pod.yaml.out
819
kubectl -n $(NAMESPACE) apply -f admin-pod.yaml.out

‎admin-pod/admin-pod.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ spec:
3131
ln -s /ssl-config/ssl-config.curlrc $HOME/.curlrc
3232
ln -s /sql-config/sql-config.cnf $HOME/.my.cnf
3333
while true; do sleep 1000; done
34-
image: {{ service_base_image.image }}
34+
image: {{ admin_pod_image.image }}
3535
resources:
3636
requests:
3737
cpu: "1m"

‎auth/Dockerfile

+24-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,29 @@
1-
FROM {{ service_base_image.image }}
1+
FROM {{ hail_ubuntu_image.image }}
2+
3+
COPY hail/python/hailtop/pinned-requirements.txt hailtop-requirements.txt
4+
COPY gear/pinned-requirements.txt gear-requirements.txt
5+
COPY web_common/pinned-requirements.txt web_common-requirements.txt
6+
COPY auth/pinned-requirements.txt auth-requirements.txt
7+
RUN hail-pip-install \
8+
-r hailtop-requirements.txt \
9+
-r gear-requirements.txt \
10+
-r web_common-requirements.txt \
11+
-r auth-requirements.txt
12+
13+
COPY hail/python/setup-hailtop.py /hailtop/setup.py
14+
COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in
15+
COPY hail/python/hailtop /hailtop/hailtop/
16+
17+
COPY gear/setup.py /gear/setup.py
18+
COPY gear/gear /gear/gear/
19+
20+
COPY web_common/setup.py web_common/MANIFEST.in /web_common/
21+
COPY web_common/web_common /web_common/web_common/
222

323
COPY auth/setup.py auth/MANIFEST.in /auth/
424
COPY auth/auth /auth/auth/
5-
RUN hail-pip-install /auth && rm -rf /auth
25+
26+
RUN hail-pip-install /hailtop /gear /web_common /auth && \
27+
rm -rf /hailtop /gear /web_common /auth
628

729
EXPOSE 5000

‎auth/Makefile

+2-19
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,14 @@
11
include ../config.mk
22

3-
AUTH_IMAGE := $(DOCKER_PREFIX)/auth:$(TOKEN)
4-
5-
EXTRA_PYTHONPATH := ../hail/python:../gear:../web_common
6-
PYTHON := PYTHONPATH=$${PYTHONPATH:+$${PYTHONPATH}:}$(EXTRA_PYTHONPATH) python3
7-
BLACK := $(PYTHON) -m black . --line-length=120 --skip-string-normalization
8-
9-
.PHONY: check
10-
check:
11-
$(PYTHON) -m flake8 auth
12-
$(PYTHON) -m pylint --rcfile ../pylintrc auth --score=n
13-
$(PYTHON) -m isort . --check-only --diff
14-
$(BLACK) --check --diff
15-
curlylint .
16-
bash ../check-sql.sh
17-
183
.PHONY: build
194
build:
20-
$(MAKE) -C ../docker service-base
21-
python3 ../ci/jinja2_render.py '{"service_base_image":{"image":"'$$(cat ../docker/service-base-image-ref)'"}}' Dockerfile Dockerfile.out
22-
../docker-build.sh .. auth/Dockerfile.out $(AUTH_IMAGE)
5+
$(MAKE) -C .. auth-image
236

247
.PHONY: deploy
258
deploy: build
269
! [ -z $(NAMESPACE) ] # call this like: make deploy NAMESPACE=default
2710
kubectl -n $(NAMESPACE) apply -f auth-driver-service-account.yaml
28-
python3 ../ci/jinja2_render.py '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"default_ns":{"name":"$(NAMESPACE)"},"scope":"$(SCOPE)","auth_image":{"image":"$(AUTH_IMAGE)"},"auth_database":{"user_secret_name":"sql-auth-user-config"}}' deployment.yaml deployment.yaml.out
11+
python3 ../ci/jinja2_render.py '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"scope":"$(SCOPE)","default_ns":{"name":"$(NAMESPACE)"},"auth_image":{"image":"'$$(cat ../auth-image)'"},"auth_database":{"user_secret_name":"sql-auth-user-config"}}' deployment.yaml deployment.yaml.out
2912
kubectl -n $(NAMESPACE) apply -f deployment.yaml.out
3013

3114
.PHONY:

‎auth/auth/driver/driver.py

+7-7
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import os
66
import random
77
import secrets
8-
from typing import Optional
8+
from typing import Any, Awaitable, Callable, Dict, List, Optional
99

1010
import aiohttp
1111
import kubernetes_asyncio.client
@@ -390,9 +390,9 @@ async def _delete(self, user, billing_project):
390390
if e.status == 403 and 'Unknown Hail Batch billing project' in e.body:
391391
return
392392
raise
393-
else:
394-
if bp['status'] == 'closed':
395-
await self.batch_client.reopen_billing_project(billing_project)
393+
394+
if bp['status'] == 'closed':
395+
await self.batch_client.reopen_billing_project(billing_project)
396396

397397
try:
398398
await self.batch_client.remove_user(user, billing_project)
@@ -521,7 +521,7 @@ async def _create_user(app, user, skip_trial_bp, cleanup):
521521

522522

523523
async def create_user(app, user, skip_trial_bp=False):
524-
cleanup = []
524+
cleanup: List[Callable[[], Awaitable[None]]] = []
525525
try:
526526
await _create_user(app, user, skip_trial_bp, cleanup)
527527
except Exception:
@@ -610,7 +610,7 @@ async def update_users(app):
610610

611611

612612
async def async_main():
613-
app = {}
613+
app: Dict[str, Any] = {}
614614

615615
user_creation_loop = None
616616
try:
@@ -629,7 +629,7 @@ async def async_main():
629629

630630
app['identity_client'] = get_identity_client()
631631

632-
app['batch_client'] = await bc.aioclient.BatchClient.create(None)
632+
app['batch_client'] = await bc.aioclient.BatchClient.create('')
633633

634634
users_changed_event = asyncio.Event()
635635
app['users_changed_event'] = users_changed_event

‎auth/auth/flow.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -92,17 +92,17 @@ def initiate_flow(self, redirect_uri: str) -> dict:
9292
}
9393

9494
def receive_callback(self, request: aiohttp.web.Request, flow_dict: dict) -> FlowResult:
95-
query_dict = urllib.parse.parse_qs(request.query_string)
96-
query_dict = {k: v[0] for k, v in query_dict.items()}
95+
query_key_to_list_of_values = urllib.parse.parse_qs(request.query_string)
96+
query_dict = {k: v[0] for k, v in query_key_to_list_of_values.items()}
9797

9898
token = self._client.acquire_token_by_auth_code_flow(flow_dict['flow'], query_dict)
9999

100100
if 'error' in token:
101-
raise Exception(f'{token}')
101+
raise ValueError(token)
102102

103103
tid = token['id_token_claims']['tid']
104104
if tid != self._tenant_id:
105-
raise Exception('invalid tenant id')
105+
raise ValueError('invalid tenant id')
106106

107107
return FlowResult(token['id_token_claims']['oid'], token['id_token_claims']['preferred_username'], token)
108108

0 commit comments

Comments
 (0)