[hailctl] Fix dependency installation for hailctl dataproc clusters (#12510)

* [hailctl] Fix dependency installation for hailctl dataproc clusters

* make the pinned requirements files actual targets

* strip the spaces from each dependency

* fix hailctl dataproc package installation

* require the user regenerate pinned-requirements for install-on-cluster if out of date

* install pip-tools

* pip installing pip-tools puts it in the user local bin which is not on path in dataproc

* fix #s

* just have the user sudo

* add permission again

* fix requirements parsing before installation

* put lockfile generation into a separate file

* fix for dataproc

* remove dependency from deploy.yaml

* remove targets but ensure compatibility

* undo requirements test change

* fix requirements

* fix'

* add file to build.yaml

* add more inputs in build.yaml

* make singular

Author: daniel-goldstein

build.yaml

@@ -826,6 +826,8 @@ steps:
         to: /io/repo/README.md
       - from: /repo/hail
         to: /io/repo/hail
+      - from: /repo/check_pip_requirements.sh
+        to: /io/repo/check_pip_requirements.sh
     dependsOn:
       - hail_build_image
       - merge_code
@@ -848,6 +850,8 @@ steps:
         to: /io/repo/README.md
       - from: /repo/hail
         to: /io/repo/hail
+      - from: /repo/check_pip_requirements.sh
+        to: /io/repo/check_pip_requirements.sh
     dependsOn:
       - hail_build_image
       - merge_code
@@ -871,6 +875,8 @@ steps:
         to: /io/repo/README.md
       - from: /repo/hail
         to: /io/repo/hail
+      - from: /repo/check_pip_requirements.sh
+        to: /io/repo/check_pip_requirements.sh
     outputs:
       - from: /io/repo/hail/build/deploy/dist/wheel-container.tar
         to: /wheel-for-azure-container.tar
@@ -2790,9 +2796,9 @@ steps:
 
       cat >file-with-args.sh <<EOF
       set -ex
-      
+
       [[ $# -eq 2 ]]
-      
+
       cat foo
       echo "Hello World! $1 $2"
       EOF

check_pip_requirements.sh

@@ -8,12 +8,15 @@ new_pinned=$(mktemp)
 pinned_no_comments=$(mktemp)
 new_pinned_no_comments=$(mktemp)
 
-pip-compile --quiet $reqs $pinned --output-file=$new_pinned
+PATH="$PATH:$HOME/.local/bin" pip-compile --quiet $reqs $pinned --output-file=$new_pinned
 # Get rid of comments that might differ despite requirements being the same
 cat $pinned | sed '/#/d' > $pinned_no_comments
 cat $new_pinned | sed '/#/d' > $new_pinned_no_comments
 diff $pinned_no_comments $new_pinned_no_comments || {
     echo '>>> up-to-date pinned requirements <<<'
     cat $new_pinned
+    echo '--------------------------------------'
+    echo "$pinned is no longer up to date with $reqs"
+    echo "Please regenerate the pinned requirements file."
     exit 1
 }

docker/Makefile

@@ -40,13 +40,7 @@ deploy: build
 
 .PHONY: generate-pip-lockfile
 generate-pip-lockfile:
-	# Some service dependencies are linux-specific and the lockfile
-	# would differ when generated on MacOS, so we generate the lockfile
-	# on a linux image.
-	docker run --rm -it \
-		-v $(HAIL):/hail \
-		python:3.7-slim \
-		/bin/bash -c "pip install pip-tools && cd hail/docker && pip-compile --upgrade requirements.txt --output-file=linux-pinned-requirements.txt"
+	HAIL_HAIL_DIR=$(HAIL) ../generate_pip_lockfile.sh requirements.txt linux-pinned-requirements.txt
 
 .PHONY: clean
 clean:

generate_pip_lockfile.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -ex
+
+reqs=$1
+pinned_reqs=$2
+
+# Some service dependencies are linux-specific and the lockfile
+# would differ when generated on MacOS, so we generate the lockfile
+# on a linux image.
+if [[ "$(uname)" == 'Linux' ]]; then
+    # `pip install pip-tools` on dataproc by default installs into the
+    # user's local bin which is not on the PATH
+    PATH="$PATH:$HOME/.local/bin" pip-compile --upgrade $reqs --output-file=$pinned_reqs
+else
+	docker run --rm -it \
+        -v ${HAIL_HAIL_DIR}:/hail \
+		python:3.7-slim \
+		/bin/bash -c "pip install pip-tools && cd /hail && pip-compile --upgrade $reqs --output-file=$pinned_reqs"
+fi

hail/Makefile

@@ -268,14 +268,31 @@ resources := $(wildcard python/hailtop/hailctl/dataproc/resources/*)
 $(eval $(call ENV_VAR,cloud_base))
 $(eval $(call ENV_VAR,wheel_cloud_path))
 
+
+.PHONY: generate-pip-lockfiles
+generate-pip-lockfiles:
+	HAIL_HAIL_DIR=$(HAIL_HAIL_DIR) \
+	../generate_pip_lockfile.sh python/hailtop/requirements.txt python/hailtop/pinned-requirements.txt && \
+	HAIL_HAIL_DIR=$(HAIL_HAIL_DIR) \
+	../generate_pip_lockfile.sh python/requirements.txt python/pinned-requirements.txt && \
+	HAIL_HAIL_DIR=$(HAIL_HAIL_DIR) \
+	../generate_pip_lockfile.sh python/dev/requirements.txt python/dev/pinned-requirements.txt
+
+
+.PHONY: check-pip-lockfile
+check-pip-lockfile:
+	$(PIP) install pip-tools && bash ../check_pip_requirements.sh python/requirements.txt python/pinned-requirements.txt
+
+
 python/hailtop/hailctl/deploy.yaml: env/cloud_base env/wheel_cloud_path
-python/hailtop/hailctl/deploy.yaml: $(resources) python/requirements.txt
+python/hailtop/hailctl/deploy.yaml: $(resources) check-pip-lockfile
 	rm -f $@
 	echo "dataproc:" >> $@
 	for FILE in $(notdir $(resources)); do \
 	  echo "  $$FILE: $(cloud_base)/$$FILE" >> $@ || exit 1; done
 	echo "  wheel: $(wheel_cloud_path)" >> $@
-	echo "  pip_dependencies: $(shell cat python/requirements.txt | sed '/^[[:blank:]]*#/d;s/#.*//' | grep -v pyspark | tr "\n" "|||")" >> $@
+	printf "  pip_dependencies: " >> $@
+	cat python/pinned-requirements.txt | sed '/^[[:blank:]]*#/d;s/#.*//' | grep -v pyspark | tr "\n" "|||" | tr -d '[:space:]' >> $@
 
 .PHONY: upload-artifacts
 upload-artifacts: $(WHEEL)
@@ -324,8 +341,8 @@ install: $(WHEEL)
 	hailctl config set query/backend spark
 
 .PHONY: install-on-cluster
-install-on-cluster: $(WHEEL)
-	sed '/^pyspark/d' python/requirements.txt | grep -v '^#' | tr '\n' '\0' | xargs -0 $(PIP) install -U
+install-on-cluster: $(WHEEL) check-pip-lockfiles
+	sed '/^pyspark/d' python/pinned-requirements.txt | grep -v -e '^[[:space:]]*#' -e '^$$' | tr '\n' '\0' | xargs -0 $(PIP) install -U
 	-$(PIP) uninstall -y hail
 	$(PIP) install $(WHEEL) --no-deps
 	hailctl config set query/backend spark
@@ -372,19 +389,6 @@ install-dev-deps:
 install-deps: install-dev-deps
 	sed "s/^pyspark.*/pyspark==$(SPARK_VERSION)/" python/requirements.txt | xargs $(PIP) install -U
 
-.PHONY: generate-pip-lockfile
-generate-pip-lockfile:
-	docker run --rm -it \
-		-v $(HAIL_HAIL_DIR):/hail \
-		python:3.7-slim \
-		/bin/bash -c "pip install pip-tools && cd /hail && \
-	pip-compile --upgrade python/hailtop/requirements.txt \
-		--output-file=python/hailtop/pinned-requirements.txt && \
-	pip-compile --upgrade python/requirements.txt \
-		--output-file=python/pinned-requirements.txt && \
-	pip-compile --upgrade python/dev/requirements.txt \
-		--output-file=python/dev/pinned-requirements.txt"
-
 .PHONY: benchmark
 benchmark: $(WHEEL)
 	HAIL_WHEEL=../hail/$(WHEEL) HAIL_PIP_VERSION=$(HAIL_PIP_VERSION) $(MAKE) -C ../benchmark benchmark

hail/python/dev/pinned-requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.7
+# by the following command:
 #
 #    pip-compile --output-file=python/dev/pinned-requirements.txt python/dev/requirements.txt
 #
@@ -50,13 +50,13 @@ backcall==0.2.0
     # via ipython
 beautifulsoup4==4.11.1
     # via nbconvert
-black==22.10.0
+black==22.12.0
     # via -r python/dev/requirements.txt
 bleach==5.0.1
     # via nbconvert
-boto3==1.26.20
+boto3==1.26.26
     # via -r python/dev/../hailtop/requirements.txt
-botocore==1.29.20
+botocore==1.29.26
     # via
     #   -r python/dev/../hailtop/requirements.txt
     #   boto3
@@ -124,7 +124,7 @@ frozenlist==1.3.3
     #   aiosignal
 fswatch==0.1.1
     # via -r python/dev/requirements.txt
-google-api-core==2.10.2
+google-api-core==2.11.0
     # via
     #   google-cloud-core
     #   google-cloud-storage
@@ -136,7 +136,7 @@ google-auth==2.14.1
     #   google-cloud-storage
 google-cloud-core==2.3.2
     # via google-cloud-storage
-google-cloud-storage==2.6.0
+google-cloud-storage==2.7.0
     # via -r python/dev/../hailtop/requirements.txt
 google-crc32c==1.5.0
     # via google-resumable-media
@@ -163,7 +163,7 @@ importlib-metadata==3.10.1
     #   nbformat
     #   pluggy
     #   pytest
-importlib-resources==5.10.0
+importlib-resources==5.10.1
     # via jsonschema
 iniconfig==1.1.1
     # via pytest
@@ -185,7 +185,7 @@ ipython-genutils==0.2.0
     #   nbclassic
     #   notebook
     #   qtconsole
-ipywidgets==8.0.2
+ipywidgets==8.0.3
     # via jupyter
 isodate==0.6.1
     # via msrest
@@ -214,7 +214,7 @@ jsonschema==4.17.3
     # via nbformat
 jupyter==1.0.0
     # via -r python/dev/requirements.txt
-jupyter-client==7.4.7
+jupyter-client==7.4.8
     # via
     #   ipykernel
     #   jupyter-console
@@ -241,7 +241,7 @@ jupyter-server==1.23.3
     #   notebook-shim
 jupyterlab-pygments==0.2.2
     # via nbconvert
-jupyterlab-widgets==3.0.3
+jupyterlab-widgets==3.0.4
     # via ipywidgets
 lazy-object-proxy==1.8.0
     # via astroid
@@ -267,7 +267,7 @@ msal-extensions==1.0.0
     # via azure-identity
 msrest==0.7.1
     # via azure-storage-blob
-multidict==6.0.2
+multidict==6.0.3
     # via
     #   aiohttp
     #   yarl
@@ -281,7 +281,7 @@ nbclassic==0.4.8
     # via notebook
 nbclient==0.7.2
     # via nbconvert
-nbconvert==7.2.5
+nbconvert==7.2.6
     # via
     #   jupyter
     #   jupyter-server
@@ -311,9 +311,9 @@ notebook-shim==0.2.2
     # via nbclassic
 oauthlib==3.2.2
     # via requests-oauthlib
-orjson==3.8.2
+orjson==3.8.3
     # via -r python/dev/../hailtop/requirements.txt
-packaging==21.3
+packaging==22.0
     # via
     #   ipykernel
     #   jupyter-server
@@ -337,7 +337,7 @@ pickleshare==0.7.5
     # via ipython
 pkgutil-resolve-name==1.3.10
     # via jsonschema
-platformdirs==2.5.4
+platformdirs==2.6.0
     # via
     #   black
     #   pylint
@@ -350,7 +350,7 @@ prometheus-client==0.15.0
     #   jupyter-server
     #   nbclassic
     #   notebook
-prompt-toolkit==3.0.33
+prompt-toolkit==3.0.36
     # via
     #   ipython
     #   jupyter-console
@@ -391,10 +391,8 @@ pyjwt[crypto]==2.6.0
     # via
     #   -c python/dev/../requirements.txt
     #   msal
-pylint==2.15.7
+pylint==2.15.8
     # via -r python/dev/requirements.txt
-pyparsing==3.0.9
-    # via packaging
 pyrsistent==0.19.2
     # via jsonschema
 pytest==7.2.0
@@ -406,7 +404,7 @@ pytest==7.2.0
     #   pytest-instafail
     #   pytest-metadata
     #   pytest-xdist
-pytest-asyncio==0.20.2
+pytest-asyncio==0.20.3
     # via -r python/dev/requirements.txt
 pytest-forked==1.4.0
     # via pytest-xdist
@@ -504,7 +502,7 @@ sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
 tabulate==0.9.0
     # via -r python/dev/../hailtop/requirements.txt
-terminado==0.17.0
+terminado==0.17.1
     # via
     #   jupyter-server
     #   nbclassic
@@ -529,7 +527,7 @@ tornado==6.2
     #   nbclassic
     #   notebook
     #   terminado
-traitlets==5.6.0
+traitlets==5.7.0
     # via
     #   ipykernel
     #   ipython
@@ -550,7 +548,7 @@ typed-ast==1.5.4
     #   astroid
     #   black
     #   mypy
-types-chardet==5.0.4
+types-chardet==5.0.4.1
     # via -r python/dev/requirements.txt
 types-decorator==5.1.8.1
     # via -r python/dev/requirements.txt
@@ -564,7 +562,7 @@ types-pyyaml==6.0.12.2
     # via -r python/dev/requirements.txt
 types-requests==2.28.11.5
     # via -r python/dev/requirements.txt
-types-setuptools==65.6.0.1
+types-setuptools==65.6.0.2
     # via -r python/dev/requirements.txt
 types-six==1.16.21.4
     # via -r python/dev/requirements.txt
@@ -606,11 +604,11 @@ webencodings==0.5.1
     #   tinycss2
 websocket-client==1.4.2
     # via jupyter-server
-widgetsnbextension==4.0.3
+widgetsnbextension==4.0.4
     # via ipywidgets
 wrapt==1.14.1
     # via astroid
-yarl==1.8.1
+yarl==1.8.2
     # via aiohttp
 zipp==3.11.0
     # via

hail/python/hailtop/hailctl/dataproc/resources/init_notebook.py

@@ -38,7 +38,6 @@ def mkdir_if_not_exists(path):
         'setuptools',
         'mkl<2020',
         'lxml<5',
-        'google-cloud-storage==1.25.*',
         'https://github.com/hail-is/jgscm/archive/v0.1.12+hail.zip',
         'ipykernel==4.10.*',
         'ipywidgets==7.5.*',