[RFC] use pip sync instead of pip install for pinned dependencies

Dockerfile

@@ -6,12 +6,8 @@ RUN apt-get update && apt-get install -y git gcc
 
 RUN apt install -y default-jre
 
-RUN python -m pip install -U pip
-# libcrypto fix oct 2023; should be able to remove sometime after that
-RUN python -m pip uninstall oscrypto -y
-RUN python -m pip install git+https://github.com/wbond/oscrypto.git@d5f3437ed24257895ae1edd9e503cfb352e635a8
 RUN python -m pip install -U uv
 
 ADD . .
 
-RUN uv pip install --system -e .
+RUN uv pip sync requirements.txt --system

Makefile

@@ -5,6 +5,16 @@ locally: manifest
 clean:
 	rm -rf ~/.dagster_home; mkdir ~/.dagster_home; cp dagster.yaml ~/.dagster_home/dagster.yaml
 
+run_docker_update:
+	docker build - < Update-Docker-Requirements/Dockerfile -t update-docker-python-dependencies && docker run --rm  -v ./:/app -w /app update-docker-python-dependencies /bin/bash
+
+update_python_packages:
+	uv pip compile --upgrade --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in --system;
+	uv pip compile --upgrade --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py --system;
+	uv pip compile --upgrade --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in --system;
+	uv pip compile --upgrade --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py --system;
+	uv pip compile --upgrade --output-file=requirements.txt setup.py --system;
+
 manifest:
 	dbt parse --project-dir=dbt_project --target BRANCH
 

README.md

@@ -46,3 +46,7 @@ This repository uses Dagster Cloud Hybrid architecture with GitHub Actions to pr
 - The open PR in this repository shows how Dagster supports full integration testing with a *branch deployment*, in this case the PR is code for a second "competing" model. This change also highlights how you can test dependency changes. This cxapability is also implemented in the GitHub Action in this repo.
 
 *Dev Notes in the Repo Wiki*
+
+## Updating Python Packages
+
+We use a combination of `requirements.in` and `setup.py` to track necessary python pacakges. To bump package versions, run `make run_docker_update` and then commit the updated `requirements.txt` files. This allows us to control when we upgrade Dagster versions, but tries to mimimize the friction involved with doing so. You will need Docker set up on your computer to run it.

Update-Docker-Requirements/Dockerfile

@@ -0,0 +1,9 @@
+FROM amd64/python:3.10-buster
+
+WORKDIR /opt/dagster/app
+
+RUN apt-get update && apt-get install -y make
+
+RUN python -m pip install -U uv
+
+ENTRYPOINT make update_python_packages

hooli-demo-assets/Dockerfile

@@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app
 
 ADD . .
 
-RUN pip install -e .
+RUN python -m pip install -U uv
+RUN uv pip sync requirements.txt --system

hooli-demo-assets/requirements.txt

@@ -0,0 +1,223 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py --system
+alembic==1.13.1
+    # via dagster
+annotated-types==0.6.0
+    # via pydantic
+astunparse==1.6.3
+    # via dlt
+certifi==2024.2.2
+    # via requests
+cffi==1.16.0
+    # via cryptography
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via
+    #   dagster
+    #   dlt
+    #   typer
+coloredlogs==14.0
+    # via dagster
+croniter==2.0.3
+    # via dagster
+cryptography==42.0.5
+    # via pyjwt
+dagster==1.7.2
+    # via
+    #   dagster-cloud
+    #   dagster-embedded-elt
+dagster-cloud==1.7.2
+dagster-cloud-cli==1.7.2
+    # via dagster-cloud
+dagster-embedded-elt==0.23.2
+dagster-pipes==1.7.2
+    # via dagster
+dlt==0.4.8
+    # via dagster-embedded-elt
+docstring-parser==0.16
+    # via dagster
+fsspec==2024.3.1
+    # via
+    #   dlt
+    #   universal-pathlib
+gitdb==4.0.11
+    # via gitpython
+github3-py==4.0.1
+    # via dagster-cloud-cli
+gitpython==3.1.43
+    # via dlt
+giturlparse==0.12.0
+    # via dlt
+greenlet==3.0.3
+    # via sqlalchemy
+grpcio==1.62.2
+    # via
+    #   dagster
+    #   grpcio-health-checking
+grpcio-health-checking==1.62.2
+    # via dagster
+hexbytes==1.2.0
+    # via dlt
+humanfriendly==10.0
+    # via coloredlogs
+humanize==4.9.0
+    # via dlt
+idna==3.7
+    # via requests
+jinja2==3.1.3
+    # via dagster
+jsonpath-ng==1.6.1
+    # via dlt
+makefun==1.15.2
+    # via dlt
+mako==1.3.3
+    # via alembic
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via
+    #   jinja2
+    #   mako
+mdurl==0.1.2
+    # via markdown-it-py
+orjson==3.9.10
+    # via dlt
+packaging==24.0
+    # via
+    #   dagster
+    #   dagster-cloud-cli
+    #   dlt
+pathvalidate==3.2.0
+    # via dlt
+pendulum==3.0.0
+    # via
+    #   dagster
+    #   dlt
+pex==2.3.1
+    # via dagster-cloud
+ply==3.11
+    # via jsonpath-ng
+prompt-toolkit==3.0.36
+    # via questionary
+protobuf==4.25.3
+    # via
+    #   dagster
+    #   grpcio-health-checking
+pycparser==2.22
+    # via cffi
+pydantic==2.7.0
+    # via dagster
+pydantic-core==2.18.1
+    # via pydantic
+pygments==2.17.2
+    # via rich
+pyjwt==2.8.0
+    # via github3-py
+python-dateutil==2.9.0.post0
+    # via
+    #   croniter
+    #   dagster
+    #   github3-py
+    #   pendulum
+    #   time-machine
+python-dotenv==1.0.1
+    # via dagster
+pytz==2024.1
+    # via
+    #   croniter
+    #   dagster
+    #   dlt
+pyyaml==6.0.1
+    # via
+    #   dagster
+    #   dagster-cloud-cli
+    #   dlt
+questionary==2.0.1
+    # via
+    #   dagster-cloud
+    #   dagster-cloud-cli
+requests==2.31.0
+    # via
+    #   dagster
+    #   dagster-cloud
+    #   dagster-cloud-cli
+    #   dlt
+    #   github3-py
+requirements-parser==0.9.0
+    # via dlt
+rich==13.7.1
+    # via
+    #   dagster
+    #   typer
+semver==3.0.2
+    # via dlt
+setuptools==69.5.1
+    # via
+    #   dagster
+    #   dlt
+shellingham==1.5.4
+    # via typer
+simplejson==3.19.2
+    # via dlt
+six==1.16.0
+    # via
+    #   astunparse
+    #   python-dateutil
+sling==1.2.5
+    # via dagster-embedded-elt
+sling-linux-amd64==1.2.5
+    # via sling
+smmap==5.0.1
+    # via gitdb
+sqlalchemy==2.0.29
+    # via
+    #   alembic
+    #   dagster
+structlog==24.1.0
+    # via dagster
+tabulate==0.9.0
+    # via dagster
+tenacity==8.2.3
+    # via dlt
+time-machine==2.14.1
+    # via pendulum
+tomli==2.0.1
+    # via dagster
+tomlkit==0.12.4
+    # via dlt
+toposort==1.10
+    # via dagster
+tqdm==4.66.2
+    # via dagster
+typer==0.12.3
+    # via
+    #   dagster-cloud
+    #   dagster-cloud-cli
+types-setuptools==69.5.0.20240415
+    # via requirements-parser
+typing-extensions==4.11.0
+    # via
+    #   alembic
+    #   dagster
+    #   dlt
+    #   pydantic
+    #   pydantic-core
+    #   sqlalchemy
+    #   typer
+tzdata==2024.1
+    # via
+    #   dlt
+    #   pendulum
+universal-pathlib==0.2.2
+    # via dagster
+uritemplate==4.1.1
+    # via github3-py
+urllib3==2.2.1
+    # via requests
+watchdog==4.0.0
+    # via dagster
+wcwidth==0.2.13
+    # via prompt-toolkit
+wheel==0.43.0
+    # via astunparse

hooli_basics/Dockerfile

@@ -1,7 +1,8 @@
-FROM python:3.8-slim
+FROM python:3.11-slim
 
 WORKDIR /opt/dagster/app
 
 ADD . .
 
-RUN pip install -r requirements.txt
+RUN python -m pip install -U uv
+RUN uv pip sync requirements.txt --system

hooli_basics/requirements.in

@@ -0,0 +1,5 @@
+pandas
+html5lib
+scikit-learn
+dagster
+dagster-cloud