From d55b0dd838da0fe8722f3fab84671522158a3785 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 19 Apr 2024 10:24:17 -0400 Subject: [PATCH 1/7] use pip sync instead of pip install --- Dockerfile | 6 +- Makefile | 7 + README.md | 4 + hooli-demo-assets/Dockerfile | 3 +- hooli-demo-assets/requirements.txt | 221 ++++++++ hooli_basics/Dockerfile | 3 +- hooli_basics/requirements.in | 5 + hooli_basics/requirements.txt | 185 +++++- hooli_batch_enrichment/Dockerfile | 3 +- hooli_batch_enrichment/requirements.txt | 175 ++++++ hooli_snowflake_insights/Dockerfile | 3 +- hooli_snowflake_insights/requirements.in | 6 + hooli_snowflake_insights/requirements.txt | 319 ++++++++++- requirements.txt | 649 ++++++++++++++++++++++ 14 files changed, 1569 insertions(+), 20 deletions(-) create mode 100644 hooli-demo-assets/requirements.txt create mode 100644 hooli_basics/requirements.in create mode 100644 hooli_batch_enrichment/requirements.txt create mode 100644 hooli_snowflake_insights/requirements.in create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index a6dde204..717c56f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,12 +6,8 @@ RUN apt-get update && apt-get install -y git gcc RUN apt install -y default-jre -RUN python -m pip install -U pip -# libcrypto fix oct 2023; should be able to remove sometime after that -RUN python -m pip uninstall oscrypto -y -RUN python -m pip install git+https://github.com/wbond/oscrypto.git@d5f3437ed24257895ae1edd9e503cfb352e635a8 RUN python -m pip install -U uv ADD . . -RUN uv pip install --system -e . +RUN uv pip sync requirements.txt --system diff --git a/Makefile b/Makefile index d7fd0161..47f8be04 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,13 @@ locally: manifest clean: rm -rf ~/.dagster_home; mkdir ~/.dagster_home; cp dagster.yaml ~/.dagster_home/dagster.yaml +update_python_packages: + uv pip compile --upgrade --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in; + uv pip compile --upgrade --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py; + uv pip compile --upgrade --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in; + uv pip compile --upgrade --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py; + uv pip compile --upgrade --output-file=requirements.txt setup.py; + manifest: dbt parse --project-dir=dbt_project --target BRANCH diff --git a/README.md b/README.md index 6c2b3143..9f3a14e7 100644 --- a/README.md +++ b/README.md @@ -46,3 +46,7 @@ This repository uses Dagster Cloud Hybrid architecture with GitHub Actions to pr - The open PR in this repository shows how Dagster supports full integration testing with a *branch deployment*, in this case the PR is code for a second "competing" model. This change also highlights how you can test dependency changes. This cxapability is also implemented in the GitHub Action in this repo. *Dev Notes in the Repo Wiki* + +## Updating Python Packages + +We use a combination of `requirements.in` and `setup.py` to track necessary python pacakges. To bump package versions, run `make update_python_packages` and then commit the updated `requirements.txt` files. This allows us to control when we upgrade Dagster versions, but tries to mimimize the friction involved with doing so. \ No newline at end of file diff --git a/hooli-demo-assets/Dockerfile b/hooli-demo-assets/Dockerfile index 1c263839..c50b894e 100644 --- a/hooli-demo-assets/Dockerfile +++ b/hooli-demo-assets/Dockerfile @@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app ADD . . -RUN pip install -e . \ No newline at end of file +RUN python -m pip install -U uv +RUN uv pip sync requirements.txt --system \ No newline at end of file diff --git a/hooli-demo-assets/requirements.txt b/hooli-demo-assets/requirements.txt new file mode 100644 index 00000000..8c1f10d3 --- /dev/null +++ b/hooli-demo-assets/requirements.txt @@ -0,0 +1,221 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +astunparse==1.6.3 + # via dlt +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # dagster + # dlt + # typer +coloredlogs==14.0 + # via dagster +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via pyjwt +dagster==1.7.2 + # via + # dagster-cloud + # dagster-embedded-elt +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-embedded-elt==0.23.2 +dagster-pipes==1.7.2 + # via dagster +dlt==0.4.8 + # via dagster-embedded-elt +docstring-parser==0.16 + # via dagster +fsspec==2024.3.1 + # via + # dlt + # universal-pathlib +gitdb==4.0.11 + # via gitpython +github3-py==4.0.1 + # via dagster-cloud-cli +gitpython==3.1.43 + # via dlt +giturlparse==0.12.0 + # via dlt +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +hexbytes==1.2.0 + # via dlt +humanfriendly==10.0 + # via coloredlogs +humanize==4.9.0 + # via dlt +idna==3.7 + # via requests +jinja2==3.1.3 + # via dagster +jsonpath-ng==1.6.1 + # via dlt +makefun==1.15.2 + # via dlt +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako +mdurl==0.1.2 + # via markdown-it-py +orjson==3.9.10 + # via dlt +packaging==24.0 + # via + # dagster + # dagster-cloud-cli + # dlt +pathvalidate==3.2.0 + # via dlt +pendulum==3.0.0 + # via + # dagster + # dlt +pex==2.3.1 + # via dagster-cloud +ply==3.11 + # via jsonpath-ng +prompt-toolkit==3.0.36 + # via questionary +protobuf==4.25.3 + # via + # dagster + # grpcio-health-checking +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via dagster +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via rich +pyjwt==2.8.0 + # via github3-py +python-dateutil==2.9.0.post0 + # via + # croniter + # dagster + # github3-py + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +pytz==2024.1 + # via + # croniter + # dagster + # dlt +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli + # dlt +questionary==2.0.1 + # via + # dagster-cloud + # dagster-cloud-cli +requests==2.31.0 + # via + # dagster + # dagster-cloud + # dagster-cloud-cli + # dlt + # github3-py +requirements-parser==0.9.0 + # via dlt +rich==13.7.1 + # via + # dagster + # typer +semver==3.0.2 + # via dlt +setuptools==69.5.1 + # via + # dagster + # dlt +shellingham==1.5.4 + # via typer +simplejson==3.19.2 + # via dlt +six==1.16.0 + # via + # astunparse + # python-dateutil +sling==1.2.5 + # via dagster-embedded-elt +sling-mac-arm64==1.2.5 + # via sling +smmap==5.0.1 + # via gitdb +sqlalchemy==2.0.29 + # via + # alembic + # dagster +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via dagster +tenacity==8.2.3 + # via dlt +time-machine==2.14.1 + # via pendulum +tomli==2.0.1 + # via dagster +tomlkit==0.12.4 + # via dlt +toposort==1.10 + # via dagster +tqdm==4.66.2 + # via dagster +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli +types-setuptools==69.5.0.20240415 + # via requirements-parser +typing-extensions==4.11.0 + # via + # alembic + # dagster + # dlt + # pydantic + # pydantic-core + # sqlalchemy + # typer +tzdata==2024.1 + # via + # dlt + # pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==2.2.1 + # via requests +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.43.0 + # via astunparse diff --git a/hooli_basics/Dockerfile b/hooli_basics/Dockerfile index 33751b97..b162c9c3 100644 --- a/hooli_basics/Dockerfile +++ b/hooli_basics/Dockerfile @@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app ADD . . -RUN pip install -r requirements.txt +RUN python -m pip install -U uv +RUN uv pip sync requirements.txt --system diff --git a/hooli_basics/requirements.in b/hooli_basics/requirements.in new file mode 100644 index 00000000..cc02fb2c --- /dev/null +++ b/hooli_basics/requirements.in @@ -0,0 +1,5 @@ +pandas +html5lib +scikit-learn +dagster +dagster-cloud diff --git a/hooli_basics/requirements.txt b/hooli_basics/requirements.txt index cc02fb2c..e58560c3 100644 --- a/hooli_basics/requirements.txt +++ b/hooli_basics/requirements.txt @@ -1,5 +1,180 @@ -pandas -html5lib -scikit-learn -dagster -dagster-cloud +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # dagster + # typer +coloredlogs==14.0 + # via dagster +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via pyjwt +dagster==1.7.2 + # via dagster-cloud +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-pipes==1.7.2 + # via dagster +docstring-parser==0.16 + # via dagster +fsspec==2024.3.1 + # via universal-pathlib +github3-py==4.0.1 + # via dagster-cloud-cli +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +html5lib==1.1 +humanfriendly==10.0 + # via coloredlogs +idna==3.7 + # via requests +jinja2==3.1.3 + # via dagster +joblib==1.4.0 + # via scikit-learn +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako +mdurl==0.1.2 + # via markdown-it-py +numpy==1.26.4 + # via + # pandas + # scikit-learn + # scipy +packaging==24.0 + # via + # dagster + # dagster-cloud-cli +pandas==2.2.2 +pendulum==3.0.0 + # via dagster +pex==2.3.1 + # via dagster-cloud +prompt-toolkit==3.0.36 + # via questionary +protobuf==4.25.3 + # via + # dagster + # grpcio-health-checking +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via dagster +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via rich +pyjwt==2.8.0 + # via github3-py +python-dateutil==2.9.0.post0 + # via + # croniter + # dagster + # github3-py + # pandas + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +pytz==2024.1 + # via + # croniter + # dagster + # pandas +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli +questionary==2.0.1 + # via + # dagster-cloud + # dagster-cloud-cli +requests==2.31.0 + # via + # dagster + # dagster-cloud + # dagster-cloud-cli + # github3-py +rich==13.7.1 + # via + # dagster + # typer +scikit-learn==1.4.2 +scipy==1.13.0 + # via scikit-learn +setuptools==69.5.1 + # via dagster +shellingham==1.5.4 + # via typer +six==1.16.0 + # via + # html5lib + # python-dateutil +sqlalchemy==2.0.29 + # via + # alembic + # dagster +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via dagster +threadpoolctl==3.4.0 + # via scikit-learn +time-machine==2.14.1 + # via pendulum +tomli==2.0.1 + # via dagster +toposort==1.10 + # via dagster +tqdm==4.66.2 + # via dagster +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli +typing-extensions==4.11.0 + # via + # alembic + # dagster + # pydantic + # pydantic-core + # sqlalchemy + # typer +tzdata==2024.1 + # via + # pandas + # pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==2.2.1 + # via requests +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit +webencodings==0.5.1 + # via html5lib diff --git a/hooli_batch_enrichment/Dockerfile b/hooli_batch_enrichment/Dockerfile index a401d314..4f6238db 100644 --- a/hooli_batch_enrichment/Dockerfile +++ b/hooli_batch_enrichment/Dockerfile @@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app ADD . . -RUN pip install -e . +RUN python -m pip install -U uv +RUN uv pip sync requirements.txt --system diff --git a/hooli_batch_enrichment/requirements.txt b/hooli_batch_enrichment/requirements.txt new file mode 100644 index 00000000..c5e3af04 --- /dev/null +++ b/hooli_batch_enrichment/requirements.txt @@ -0,0 +1,175 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # dagster + # typer +coloredlogs==14.0 + # via dagster +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via pyjwt +dagster==1.7.2 + # via + # dagster-cloud + # dagster-duckdb +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-duckdb==0.23.2 +dagster-pipes==1.7.2 + # via dagster +docstring-parser==0.16 + # via dagster +duckdb==0.10.2 + # via dagster-duckdb +fsspec==2024.3.1 + # via universal-pathlib +github3-py==4.0.1 + # via dagster-cloud-cli +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +humanfriendly==10.0 + # via coloredlogs +idna==3.7 + # via requests +jinja2==3.1.3 + # via dagster +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako +mdurl==0.1.2 + # via markdown-it-py +numpy==1.26.4 + # via pandas +packaging==24.0 + # via + # dagster + # dagster-cloud-cli +pandas==2.2.2 +pendulum==3.0.0 + # via dagster +pex==2.3.1 + # via dagster-cloud +prompt-toolkit==3.0.36 + # via questionary +protobuf==4.25.3 + # via + # dagster + # grpcio-health-checking +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via dagster +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via rich +pyjwt==2.8.0 + # via github3-py +python-dateutil==2.9.0.post0 + # via + # croniter + # dagster + # github3-py + # pandas + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +pytz==2024.1 + # via + # croniter + # dagster + # pandas +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli + # responses +questionary==2.0.1 + # via + # dagster-cloud + # dagster-cloud-cli +requests==2.31.0 + # via + # dagster + # dagster-cloud + # dagster-cloud-cli + # github3-py + # responses +responses==0.25.0 +rich==13.7.1 + # via + # dagster + # typer +setuptools==69.5.1 + # via dagster +shellingham==1.5.4 + # via typer +six==1.16.0 + # via python-dateutil +sqlalchemy==2.0.29 + # via + # alembic + # dagster +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via dagster +time-machine==2.14.1 + # via pendulum +tomli==2.0.1 + # via dagster +toposort==1.10 + # via dagster +tqdm==4.66.2 + # via dagster +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli +typing-extensions==4.11.0 + # via + # alembic + # dagster + # pydantic + # pydantic-core + # sqlalchemy + # typer +tzdata==2024.1 + # via + # pandas + # pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==2.2.1 + # via + # requests + # responses +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit diff --git a/hooli_snowflake_insights/Dockerfile b/hooli_snowflake_insights/Dockerfile index 4ac0381f..5f1b82de 100644 --- a/hooli_snowflake_insights/Dockerfile +++ b/hooli_snowflake_insights/Dockerfile @@ -9,4 +9,5 @@ RUN python -m pip install git+https://github.com/wbond/oscrypto.git@d5f3437ed242 ADD . . -RUN pip install -r requirements.txt +RUN python -m pip install -U uv +RUN uv pip sync requirements.txt --system diff --git a/hooli_snowflake_insights/requirements.in b/hooli_snowflake_insights/requirements.in new file mode 100644 index 00000000..57b6f6e4 --- /dev/null +++ b/hooli_snowflake_insights/requirements.in @@ -0,0 +1,6 @@ +dagster +dagster-cloud +dagster-dbt +dagster-snowflake +gql +requests_toolbelt \ No newline at end of file diff --git a/hooli_snowflake_insights/requirements.txt b/hooli_snowflake_insights/requirements.txt index 57b6f6e4..03bb8824 100644 --- a/hooli_snowflake_insights/requirements.txt +++ b/hooli_snowflake_insights/requirements.txt @@ -1,6 +1,313 @@ -dagster -dagster-cloud -dagster-dbt -dagster-snowflake -gql -requests_toolbelt \ No newline at end of file +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in +agate==1.7.1 + # via dbt-core +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via gql +asn1crypto==1.5.1 + # via snowflake-connector-python +attrs==23.2.0 + # via + # jsonschema + # referencing +babel==2.14.0 + # via agate +backoff==2.2.1 + # via gql +certifi==2024.2.2 + # via + # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # cryptography + # dbt-core + # snowflake-connector-python +charset-normalizer==3.3.2 + # via + # requests + # snowflake-connector-python +click==8.1.7 + # via + # dagster + # dbt-core + # dbt-semantic-interfaces + # typer +colorama==0.4.6 + # via dbt-core +coloredlogs==14.0 + # via dagster +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via + # pyjwt + # pyopenssl + # snowflake-connector-python +dagster==1.7.2 + # via + # dagster-cloud + # dagster-dbt + # dagster-snowflake +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-dbt==0.23.2 +dagster-pipes==1.7.2 + # via dagster +dagster-snowflake==0.23.2 +dbt-core==1.7.13 + # via dagster-dbt +dbt-extractor==0.5.1 + # via dbt-core +dbt-semantic-interfaces==0.4.4 + # via dbt-core +docstring-parser==0.16 + # via dagster +filelock==3.13.4 + # via snowflake-connector-python +fsspec==2024.3.1 + # via universal-pathlib +github3-py==4.0.1 + # via dagster-cloud-cli +gql==3.5.0 +graphql-core==3.2.3 + # via gql +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +humanfriendly==10.0 + # via coloredlogs +idna==3.7 + # via + # anyio + # dbt-core + # requests + # snowflake-connector-python + # yarl +importlib-metadata==6.11.0 + # via dbt-semantic-interfaces +isodate==0.6.1 + # via + # agate + # dbt-core +jinja2==3.1.3 + # via + # dagster + # dagster-dbt + # dbt-core + # dbt-semantic-interfaces +jsonschema==4.21.1 + # via + # dbt-core + # dbt-semantic-interfaces +jsonschema-specifications==2023.12.1 + # via jsonschema +leather==0.4.0 + # via agate +logbook==1.5.3 + # via dbt-core +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako +mashumaro==3.12 + # via dbt-core +mdurl==0.1.2 + # via markdown-it-py +minimal-snowplow-tracker==0.0.2 + # via dbt-core +more-itertools==10.2.0 + # via dbt-semantic-interfaces +msgpack==1.0.8 + # via mashumaro +multidict==6.0.5 + # via yarl +networkx==3.3 + # via + # dagster-dbt + # dbt-core +orjson==3.10.1 + # via dagster-dbt +packaging==24.0 + # via + # dagster + # dagster-cloud-cli + # dagster-dbt + # dbt-core + # snowflake-connector-python +parsedatetime==2.6 + # via agate +pathspec==0.11.2 + # via dbt-core +pendulum==3.0.0 + # via dagster +pex==2.3.1 + # via dagster-cloud +platformdirs==4.2.0 + # via snowflake-connector-python +prompt-toolkit==3.0.36 + # via questionary +protobuf==4.25.3 + # via + # dagster + # dbt-core + # grpcio-health-checking +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via + # dagster + # dbt-semantic-interfaces +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via rich +pyjwt==2.8.0 + # via + # github3-py + # snowflake-connector-python +pyopenssl==24.1.0 + # via snowflake-connector-python +python-dateutil==2.9.0.post0 + # via + # croniter + # dagster + # dbt-semantic-interfaces + # github3-py + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +python-slugify==8.0.4 + # via agate +pytimeparse==1.1.8 + # via agate +pytz==2024.1 + # via + # croniter + # dagster + # dbt-core + # snowflake-connector-python +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli + # dbt-core + # dbt-semantic-interfaces +questionary==2.0.1 + # via + # dagster-cloud + # dagster-cloud-cli +referencing==0.34.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via + # dagster + # dagster-cloud + # dagster-cloud-cli + # dagster-dbt + # dbt-core + # github3-py + # minimal-snowplow-tracker + # requests-toolbelt + # snowflake-connector-python +requests-toolbelt==1.0.0 +rich==13.7.1 + # via + # dagster + # dagster-dbt + # typer +rpds-py==0.18.0 + # via + # jsonschema + # referencing +setuptools==69.5.1 + # via dagster +shellingham==1.5.4 + # via typer +six==1.16.0 + # via + # isodate + # minimal-snowplow-tracker + # python-dateutil +sniffio==1.3.1 + # via anyio +snowflake-connector-python==3.9.0 + # via dagster-snowflake +sortedcontainers==2.4.0 + # via snowflake-connector-python +sqlalchemy==2.0.29 + # via + # alembic + # dagster +sqlglot==23.10.0 + # via dagster-dbt +sqlglotrs==0.2.0 + # via sqlglot +sqlparse==0.5.0 + # via dbt-core +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via dagster +text-unidecode==1.3 + # via python-slugify +time-machine==2.14.1 + # via pendulum +tomli==2.0.1 + # via dagster +tomlkit==0.12.4 + # via snowflake-connector-python +toposort==1.10 + # via dagster +tqdm==4.66.2 + # via dagster +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli + # dagster-dbt +typing-extensions==4.11.0 + # via + # alembic + # dagster + # dbt-core + # dbt-semantic-interfaces + # mashumaro + # pydantic + # pydantic-core + # snowflake-connector-python + # sqlalchemy + # typer +tzdata==2024.1 + # via pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==1.26.18 + # via + # dbt-core + # requests +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit +yarl==1.9.4 + # via gql +zipp==3.18.1 + # via importlib-metadata diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..19dc6db8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,649 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=requirements.txt setup.py +agate==1.7.1 + # via + # dbt-core + # dbt-snowflake +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via gql +appnope==0.1.4 + # via ipykernel +asn1crypto==1.5.1 + # via snowflake-connector-python +asttokens==2.4.1 + # via stack-data +attrs==23.2.0 + # via + # jsonschema + # referencing +babel==2.14.0 + # via agate +backoff==2.2.1 + # via gql +beautifulsoup4==4.12.3 + # via nbconvert +bleach==6.1.0 + # via nbconvert +boto3==1.34.87 + # via dagster-aws +botocore==1.34.87 + # via + # boto3 + # s3transfer +cachetools==5.3.3 + # via google-auth +certifi==2024.2.2 + # via + # kubernetes + # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # cryptography + # dbt-core + # snowflake-connector-python +charset-normalizer==3.3.2 + # via + # requests + # snowflake-connector-python +click==8.1.7 + # via + # dagster + # databricks-cli + # dbt-core + # dbt-semantic-interfaces + # papermill + # typer +colorama==0.4.6 + # via dbt-core +coloredlogs==14.0 + # via dagster +comm==0.2.2 + # via ipykernel +contourpy==1.2.1 + # via matplotlib +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via + # pyjwt + # pyopenssl + # snowflake-connector-python +cycler==0.12.1 + # via matplotlib +dagster==1.7.2 + # via + # dagster-aws + # dagster-cloud + # dagster-databricks + # dagster-dbt + # dagster-duckdb + # dagster-duckdb-pandas + # dagster-k8s + # dagster-pyspark + # dagster-snowflake + # dagster-snowflake-pandas + # dagster-spark + # dagstermill +dagster-aws==0.23.2 +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-databricks==0.23.2 +dagster-dbt==0.23.2 +dagster-duckdb==0.23.2 + # via dagster-duckdb-pandas +dagster-duckdb-pandas==0.23.2 +dagster-k8s==0.23.2 +dagster-pipes==1.7.2 + # via + # dagster + # dagster-databricks +dagster-pyspark==0.23.2 + # via dagster-databricks +dagster-snowflake==0.23.2 + # via dagster-snowflake-pandas +dagster-snowflake-pandas==0.23.2 +dagster-spark==0.23.2 + # via dagster-pyspark +dagstermill==0.23.2 +databricks-api==0.9.0 + # via dagster-databricks +databricks-cli==0.18.0 + # via + # dagster-databricks + # databricks-api +databricks-sdk==0.8.0 + # via dagster-databricks +dbt-core==1.7.13 + # via + # dagster-dbt + # dbt-duckdb + # dbt-snowflake +dbt-duckdb==1.7.4 +dbt-extractor==0.5.1 + # via dbt-core +dbt-semantic-interfaces==0.4.4 + # via dbt-core +dbt-snowflake==1.7.3 +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +docstring-parser==0.16 + # via dagster +duckdb==0.10.2 + # via + # dagster-duckdb + # dbt-duckdb +entrypoints==0.4 + # via + # jupyter-client + # papermill +executing==2.0.1 + # via stack-data +fastjsonschema==2.19.1 + # via nbformat +filelock==3.13.4 + # via snowflake-connector-python +fonttools==4.51.0 + # via matplotlib +fsspec==2024.3.1 + # via universal-pathlib +github3-py==4.0.1 + # via dagster-cloud-cli +google-auth==2.29.0 + # via + # dagster-k8s + # kubernetes +gql==3.5.0 +graphql-core==3.2.3 + # via gql +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +html5lib==1.1 +humanfriendly==10.0 + # via coloredlogs +idna==3.7 + # via + # anyio + # dbt-core + # requests + # snowflake-connector-python + # yarl +importlib-metadata==6.11.0 + # via + # dbt-semantic-interfaces + # keyring +ipykernel==6.29.4 + # via dagstermill +ipython==8.23.0 + # via + # ipykernel + # scrapbook +ipython-genutils==0.2.0 + # via dagstermill +isodate==0.6.1 + # via + # agate + # dbt-core +jaraco-classes==3.4.0 + # via keyring +jedi==0.19.1 + # via ipython +jinja2==3.1.3 + # via + # dagster + # dagster-dbt + # dbt-core + # dbt-semantic-interfaces + # nbconvert +jmespath==1.0.1 + # via + # boto3 + # botocore +joblib==1.4.0 + # via scikit-learn +jsonschema==4.21.1 + # via + # dbt-core + # dbt-semantic-interfaces + # nbformat + # scrapbook +jsonschema-specifications==2023.12.1 + # via jsonschema +jupyter-client==7.4.9 + # via + # dagstermill + # ipykernel + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # nbclient + # nbconvert + # nbformat +jupyterlab-pygments==0.3.0 + # via nbconvert +keyring==24.3.1 + # via snowflake-connector-python +kiwisolver==1.4.5 + # via matplotlib +kubernetes==29.0.0 + # via dagster-k8s +leather==0.4.0 + # via agate +logbook==1.5.3 + # via dbt-core +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako + # nbconvert +mashumaro==3.12 + # via dbt-core +matplotlib==3.8.4 + # via + # mizani + # plotnine +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mdurl==0.1.2 + # via markdown-it-py +minimal-snowplow-tracker==0.0.2 + # via dbt-core +mistune==3.0.2 + # via nbconvert +mizani==0.9.3 + # via plotnine +more-itertools==10.2.0 + # via + # dbt-semantic-interfaces + # jaraco-classes +msgpack==1.0.8 + # via mashumaro +multidict==6.0.5 + # via yarl +nbclient==0.10.0 + # via + # nbconvert + # papermill +nbconvert==7.16.3 + # via dagstermill +nbformat==5.10.4 + # via + # nbclient + # nbconvert + # papermill +nest-asyncio==1.6.0 + # via + # ipykernel + # jupyter-client +networkx==3.3 + # via + # dagster-dbt + # dbt-core +numpy==1.26.4 + # via + # contourpy + # matplotlib + # mizani + # pandas + # patsy + # plotnine + # pyarrow + # scikit-learn + # scipy + # statsmodels +oauthlib==3.2.2 + # via + # databricks-cli + # kubernetes + # requests-oauthlib +orjson==3.10.1 + # via dagster-dbt +packaging==24.0 + # via + # dagster + # dagster-aws + # dagster-cloud-cli + # dagster-dbt + # dagstermill + # dbt-core + # ipykernel + # matplotlib + # nbconvert + # snowflake-connector-python + # statsmodels +pandas==2.0.3 + # via + # dagster-duckdb-pandas + # dagster-snowflake-pandas + # mizani + # plotnine + # scrapbook + # snowflake-connector-python + # statsmodels +pandocfilters==1.5.1 + # via nbconvert +papermill==2.5.0 + # via + # dagstermill + # scrapbook +parsedatetime==2.6 + # via agate +parso==0.8.4 + # via jedi +pathspec==0.11.2 + # via dbt-core +patsy==0.5.6 + # via + # plotnine + # statsmodels +pendulum==3.0.0 + # via dagster +pex==2.3.1 + # via dagster-cloud +pexpect==4.9.0 + # via ipython +pillow==10.3.0 + # via matplotlib +platformdirs==4.2.0 + # via + # jupyter-core + # snowflake-connector-python +plotnine==0.12.4 +prompt-toolkit==3.0.43 + # via + # ipython + # questionary +protobuf==4.25.3 + # via + # dagster + # dbt-core + # grpcio-health-checking +psutil==5.9.8 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +py4j==0.10.9.7 + # via pyspark +pyarrow==15.0.2 + # via + # scrapbook + # snowflake-connector-python +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via + # dagster + # dbt-semantic-interfaces +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via + # ipython + # nbconvert + # rich +pyjwt==2.8.0 + # via + # databricks-cli + # github3-py + # snowflake-connector-python +pyopenssl==24.1.0 + # via snowflake-connector-python +pyparsing==3.1.2 + # via matplotlib +pyspark==3.5.1 + # via dagster-pyspark +python-dateutil==2.9.0.post0 + # via + # botocore + # croniter + # dagster + # dbt-semantic-interfaces + # github3-py + # jupyter-client + # kubernetes + # matplotlib + # pandas + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +python-slugify==8.0.4 + # via agate +pytimeparse==1.1.8 + # via agate +pytz==2024.1 + # via + # croniter + # dagster + # dbt-core + # pandas + # snowflake-connector-python +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli + # dbt-core + # dbt-semantic-interfaces + # kubernetes + # papermill + # responses +pyzmq==26.0.1 + # via + # ipykernel + # jupyter-client +questionary==1.10.0 + # via + # dagster-cloud + # dagster-cloud-cli +referencing==0.34.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via + # dagster + # dagster-aws + # dagster-cloud + # dagster-cloud-cli + # dagster-dbt + # dagster-snowflake-pandas + # databricks-cli + # databricks-sdk + # dbt-core + # github3-py + # kubernetes + # minimal-snowplow-tracker + # papermill + # requests-oauthlib + # requests-toolbelt + # responses + # snowflake-connector-python +requests-oauthlib==2.0.0 + # via kubernetes +requests-toolbelt==1.0.0 +responses==0.25.0 +rich==13.7.1 + # via + # dagster + # dagster-dbt + # typer +rpds-py==0.18.0 + # via + # jsonschema + # referencing +rsa==4.9 + # via google-auth +s3transfer==0.10.1 + # via boto3 +scikit-learn==1.4.2 +scipy==1.13.0 + # via + # mizani + # plotnine + # scikit-learn + # statsmodels +scrapbook==0.5.0 + # via dagstermill +setuptools==69.5.1 + # via dagster +shellingham==1.5.4 + # via typer +six==1.16.0 + # via + # asttokens + # bleach + # databricks-cli + # html5lib + # isodate + # kubernetes + # minimal-snowplow-tracker + # patsy + # python-dateutil +sniffio==1.3.1 + # via anyio +snowflake-connector-python==3.9.0 + # via + # dagster-snowflake + # dagster-snowflake-pandas + # dbt-snowflake + # snowflake-sqlalchemy +snowflake-sqlalchemy==1.5.3 + # via dagster-snowflake-pandas +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.5 + # via beautifulsoup4 +sqlalchemy==1.4.52 + # via + # alembic + # dagster + # dagster-snowflake-pandas + # snowflake-sqlalchemy +sqlglot==23.10.0 + # via dagster-dbt +sqlglotrs==0.2.0 + # via sqlglot +sqlparse==0.5.0 + # via dbt-core +stack-data==0.6.3 + # via ipython +statsmodels==0.14.2 + # via plotnine +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via + # dagster + # databricks-cli +tenacity==8.2.3 + # via papermill +text-unidecode==1.3 + # via python-slugify +threadpoolctl==3.4.0 + # via scikit-learn +time-machine==2.14.1 + # via pendulum +tinycss2==1.2.1 + # via nbconvert +tomli==2.0.1 + # via dagster +tomlkit==0.12.4 + # via snowflake-connector-python +toposort==1.10 + # via dagster +tornado==6.4 + # via + # ipykernel + # jupyter-client +tqdm==4.66.2 + # via + # dagster + # papermill +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli + # dagster-dbt +typing-extensions==4.11.0 + # via + # alembic + # dagster + # dbt-core + # dbt-semantic-interfaces + # ipython + # mashumaro + # pydantic + # pydantic-core + # snowflake-connector-python + # typer +tzdata==2024.1 + # via + # pandas + # pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==1.26.18 + # via + # botocore + # databricks-cli + # dbt-core + # kubernetes + # requests + # responses +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # html5lib + # tinycss2 +websocket-client==1.7.0 + # via kubernetes +yarl==1.9.4 + # via gql +zipp==3.18.1 + # via importlib-metadata From 734f8d1e15a213f92d88164bfa30a14a6ba6929f Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 19 Apr 2024 12:20:03 -0400 Subject: [PATCH 2/7] bump python version --- hooli_basics/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooli_basics/Dockerfile b/hooli_basics/Dockerfile index b162c9c3..6abd7f25 100644 --- a/hooli_basics/Dockerfile +++ b/hooli_basics/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8-slim +FROM python:3.11-slim WORKDIR /opt/dagster/app From a1b5eacfca0ec05832322f499af095d8c412fdd9 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 19 Apr 2024 13:14:53 -0400 Subject: [PATCH 3/7] use a Dockerfile to generate --- Makefile | 13 ++++++++----- Update-Docker-Requirements/Dockerfile | 9 +++++++++ hooli-demo-assets/requirements.txt | 6 ++++-- hooli_basics/requirements.txt | 4 +++- hooli_batch_enrichment/requirements.txt | 4 +++- hooli_snowflake_insights/requirements.txt | 4 +++- requirements.txt | 13 ++++++++++--- 7 files changed, 40 insertions(+), 13 deletions(-) create mode 100644 Update-Docker-Requirements/Dockerfile diff --git a/Makefile b/Makefile index 47f8be04..55d4c27e 100644 --- a/Makefile +++ b/Makefile @@ -5,12 +5,15 @@ locally: manifest clean: rm -rf ~/.dagster_home; mkdir ~/.dagster_home; cp dagster.yaml ~/.dagster_home/dagster.yaml +run_docker_update: + docker build - < Update-Docker-Requirements/Dockerfile -t update-docker-python-dependencies && docker run --rm -v $(pwd):/app -w /app -it update-docker-python-dependencies /bin/bash "make update_python_packages" + update_python_packages: - uv pip compile --upgrade --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in; - uv pip compile --upgrade --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py; - uv pip compile --upgrade --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in; - uv pip compile --upgrade --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py; - uv pip compile --upgrade --output-file=requirements.txt setup.py; + uv pip compile --upgrade --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in --system; + uv pip compile --upgrade --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py --system; + uv pip compile --upgrade --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in --system; + uv pip compile --upgrade --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py --system; + uv pip compile --upgrade --output-file=requirements.txt setup.py --system; manifest: dbt parse --project-dir=dbt_project --target BRANCH diff --git a/Update-Docker-Requirements/Dockerfile b/Update-Docker-Requirements/Dockerfile new file mode 100644 index 00000000..f901ee4c --- /dev/null +++ b/Update-Docker-Requirements/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.11-slim + +WORKDIR /opt/dagster/app + +RUN apt-get update && apt-get install -y make + +RUN python -m pip install -U uv + +ENTRYPOINT make update_python_packages \ No newline at end of file diff --git a/hooli-demo-assets/requirements.txt b/hooli-demo-assets/requirements.txt index 8c1f10d3..9f22da4d 100644 --- a/hooli-demo-assets/requirements.txt +++ b/hooli-demo-assets/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py +# uv pip compile --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py --system alembic==1.13.1 # via dagster annotated-types==0.6.0 @@ -49,6 +49,8 @@ gitpython==3.1.43 # via dlt giturlparse==0.12.0 # via dlt +greenlet==3.0.3 + # via sqlalchemy grpcio==1.62.2 # via # dagster @@ -164,7 +166,7 @@ six==1.16.0 # python-dateutil sling==1.2.5 # via dagster-embedded-elt -sling-mac-arm64==1.2.5 +sling-linux-arm64==1.2.5 # via sling smmap==5.0.1 # via gitdb diff --git a/hooli_basics/requirements.txt b/hooli_basics/requirements.txt index e58560c3..1fe081d2 100644 --- a/hooli_basics/requirements.txt +++ b/hooli_basics/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in +# uv pip compile --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in --system alembic==1.13.1 # via dagster annotated-types==0.6.0 @@ -33,6 +33,8 @@ fsspec==2024.3.1 # via universal-pathlib github3-py==4.0.1 # via dagster-cloud-cli +greenlet==3.0.3 + # via sqlalchemy grpcio==1.62.2 # via # dagster diff --git a/hooli_batch_enrichment/requirements.txt b/hooli_batch_enrichment/requirements.txt index c5e3af04..6580e026 100644 --- a/hooli_batch_enrichment/requirements.txt +++ b/hooli_batch_enrichment/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py +# uv pip compile --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py --system alembic==1.13.1 # via dagster annotated-types==0.6.0 @@ -38,6 +38,8 @@ fsspec==2024.3.1 # via universal-pathlib github3-py==4.0.1 # via dagster-cloud-cli +greenlet==3.0.3 + # via sqlalchemy grpcio==1.62.2 # via # dagster diff --git a/hooli_snowflake_insights/requirements.txt b/hooli_snowflake_insights/requirements.txt index 03bb8824..7ba79258 100644 --- a/hooli_snowflake_insights/requirements.txt +++ b/hooli_snowflake_insights/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in +# uv pip compile --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in --system agate==1.7.1 # via dbt-core alembic==1.13.1 @@ -77,6 +77,8 @@ github3-py==4.0.1 gql==3.5.0 graphql-core==3.2.3 # via gql +greenlet==3.0.3 + # via sqlalchemy grpcio==1.62.2 # via # dagster diff --git a/requirements.txt b/requirements.txt index 19dc6db8..5166c206 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --output-file=requirements.txt setup.py +# uv pip compile --output-file=requirements.txt setup.py --system agate==1.7.1 # via # dbt-core @@ -10,8 +10,6 @@ annotated-types==0.6.0 # via pydantic anyio==4.3.0 # via gql -appnope==0.1.4 - # via ipykernel asn1crypto==1.5.1 # via snowflake-connector-python asttokens==2.4.1 @@ -72,6 +70,7 @@ cryptography==42.0.5 # via # pyjwt # pyopenssl + # secretstorage # snowflake-connector-python cycler==0.12.1 # via matplotlib @@ -165,6 +164,8 @@ google-auth==2.29.0 gql==3.5.0 graphql-core==3.2.3 # via gql +greenlet==3.0.3 + # via sqlalchemy grpcio==1.62.2 # via # dagster @@ -201,6 +202,10 @@ jaraco-classes==3.4.0 # via keyring jedi==0.19.1 # via ipython +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.3 # via # dagster @@ -513,6 +518,8 @@ scipy==1.13.0 # statsmodels scrapbook==0.5.0 # via dagstermill +secretstorage==3.3.3 + # via keyring setuptools==69.5.1 # via dagster shellingham==1.5.4 From ddf3be2b8c84dfc154b0d039dfe7045240478c70 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 19 Apr 2024 13:27:50 -0400 Subject: [PATCH 4/7] fix makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 55d4c27e..d090507c 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ clean: rm -rf ~/.dagster_home; mkdir ~/.dagster_home; cp dagster.yaml ~/.dagster_home/dagster.yaml run_docker_update: - docker build - < Update-Docker-Requirements/Dockerfile -t update-docker-python-dependencies && docker run --rm -v $(pwd):/app -w /app -it update-docker-python-dependencies /bin/bash "make update_python_packages" + docker build - < Update-Docker-Requirements/Dockerfile -t update-docker-python-dependencies && docker run --rm -v ./:/app -w /app update-docker-python-dependencies /bin/bash update_python_packages: uv pip compile --upgrade --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in --system; From 342356224a519bd61a918b6b3a671130fb162417 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 19 Apr 2024 13:28:05 -0400 Subject: [PATCH 5/7] update amd64 arch --- Update-Docker-Requirements/Dockerfile | 2 +- hooli-demo-assets/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Update-Docker-Requirements/Dockerfile b/Update-Docker-Requirements/Dockerfile index f901ee4c..96b91000 100644 --- a/Update-Docker-Requirements/Dockerfile +++ b/Update-Docker-Requirements/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-slim +FROM amd64/python:3.11-buster WORKDIR /opt/dagster/app diff --git a/hooli-demo-assets/requirements.txt b/hooli-demo-assets/requirements.txt index 9f22da4d..21435e8f 100644 --- a/hooli-demo-assets/requirements.txt +++ b/hooli-demo-assets/requirements.txt @@ -166,7 +166,7 @@ six==1.16.0 # python-dateutil sling==1.2.5 # via dagster-embedded-elt -sling-linux-arm64==1.2.5 +sling-linux-amd64==1.2.5 # via sling smmap==5.0.1 # via gitdb From ecdf90fd0558d85f3437c835644d21043228f218 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 19 Apr 2024 13:40:29 -0400 Subject: [PATCH 6/7] downgrade to python 3.10 --- Update-Docker-Requirements/Dockerfile | 2 +- hooli_snowflake_insights/requirements.txt | 3 +++ requirements.txt | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Update-Docker-Requirements/Dockerfile b/Update-Docker-Requirements/Dockerfile index 96b91000..87e18cb0 100644 --- a/Update-Docker-Requirements/Dockerfile +++ b/Update-Docker-Requirements/Dockerfile @@ -1,4 +1,4 @@ -FROM amd64/python:3.11-buster +FROM amd64/python:3.10-buster WORKDIR /opt/dagster/app diff --git a/hooli_snowflake_insights/requirements.txt b/hooli_snowflake_insights/requirements.txt index 7ba79258..8940098f 100644 --- a/hooli_snowflake_insights/requirements.txt +++ b/hooli_snowflake_insights/requirements.txt @@ -68,6 +68,8 @@ dbt-semantic-interfaces==0.4.4 # via dbt-core docstring-parser==0.16 # via dagster +exceptiongroup==1.2.1 + # via anyio filelock==3.13.4 # via snowflake-connector-python fsspec==2024.3.1 @@ -286,6 +288,7 @@ typer==0.12.3 typing-extensions==4.11.0 # via # alembic + # anyio # dagster # dbt-core # dbt-semantic-interfaces diff --git a/requirements.txt b/requirements.txt index 5166c206..01467b52 100644 --- a/requirements.txt +++ b/requirements.txt @@ -145,6 +145,10 @@ entrypoints==0.4 # via # jupyter-client # papermill +exceptiongroup==1.2.1 + # via + # anyio + # ipython executing==2.0.1 # via stack-data fastjsonschema==2.19.1 @@ -614,6 +618,7 @@ typer==0.12.3 typing-extensions==4.11.0 # via # alembic + # anyio # dagster # dbt-core # dbt-semantic-interfaces From 612609ba387a7409b30b70439cb31de7e46fc087 Mon Sep 17 00:00:00 2001 From: Christian Minich Date: Fri, 19 Apr 2024 13:52:32 -0400 Subject: [PATCH 7/7] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f3a14e7..a63cbfb7 100644 --- a/README.md +++ b/README.md @@ -49,4 +49,4 @@ This repository uses Dagster Cloud Hybrid architecture with GitHub Actions to pr ## Updating Python Packages -We use a combination of `requirements.in` and `setup.py` to track necessary python pacakges. To bump package versions, run `make update_python_packages` and then commit the updated `requirements.txt` files. This allows us to control when we upgrade Dagster versions, but tries to mimimize the friction involved with doing so. \ No newline at end of file +We use a combination of `requirements.in` and `setup.py` to track necessary python pacakges. To bump package versions, run `make run_docker_update` and then commit the updated `requirements.txt` files. This allows us to control when we upgrade Dagster versions, but tries to mimimize the friction involved with doing so. You will need Docker set up on your computer to run it. \ No newline at end of file