diff --git a/Dockerfile b/Dockerfile index a6dde204..717c56f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,12 +6,8 @@ RUN apt-get update && apt-get install -y git gcc RUN apt install -y default-jre -RUN python -m pip install -U pip -# libcrypto fix oct 2023; should be able to remove sometime after that -RUN python -m pip uninstall oscrypto -y -RUN python -m pip install git+https://github.com/wbond/oscrypto.git@d5f3437ed24257895ae1edd9e503cfb352e635a8 RUN python -m pip install -U uv ADD . . -RUN uv pip install --system -e . +RUN uv pip sync requirements.txt --system diff --git a/Makefile b/Makefile index d7fd0161..d090507c 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,16 @@ locally: manifest clean: rm -rf ~/.dagster_home; mkdir ~/.dagster_home; cp dagster.yaml ~/.dagster_home/dagster.yaml +run_docker_update: + docker build - < Update-Docker-Requirements/Dockerfile -t update-docker-python-dependencies && docker run --rm -v ./:/app -w /app update-docker-python-dependencies /bin/bash + +update_python_packages: + uv pip compile --upgrade --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in --system; + uv pip compile --upgrade --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py --system; + uv pip compile --upgrade --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in --system; + uv pip compile --upgrade --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py --system; + uv pip compile --upgrade --output-file=requirements.txt setup.py --system; + manifest: dbt parse --project-dir=dbt_project --target BRANCH diff --git a/README.md b/README.md index 6c2b3143..a63cbfb7 100644 --- a/README.md +++ b/README.md @@ -46,3 +46,7 @@ This repository uses Dagster Cloud Hybrid architecture with GitHub Actions to pr - The open PR in this repository shows how Dagster supports full integration testing with a *branch deployment*, in this case the PR is code for a second "competing" model. This change also highlights how you can test dependency changes. This cxapability is also implemented in the GitHub Action in this repo. *Dev Notes in the Repo Wiki* + +## Updating Python Packages + +We use a combination of `requirements.in` and `setup.py` to track necessary python pacakges. To bump package versions, run `make run_docker_update` and then commit the updated `requirements.txt` files. This allows us to control when we upgrade Dagster versions, but tries to mimimize the friction involved with doing so. You will need Docker set up on your computer to run it. \ No newline at end of file diff --git a/Update-Docker-Requirements/Dockerfile b/Update-Docker-Requirements/Dockerfile new file mode 100644 index 00000000..87e18cb0 --- /dev/null +++ b/Update-Docker-Requirements/Dockerfile @@ -0,0 +1,9 @@ +FROM amd64/python:3.10-buster + +WORKDIR /opt/dagster/app + +RUN apt-get update && apt-get install -y make + +RUN python -m pip install -U uv + +ENTRYPOINT make update_python_packages \ No newline at end of file diff --git a/hooli-demo-assets/Dockerfile b/hooli-demo-assets/Dockerfile index 1c263839..c50b894e 100644 --- a/hooli-demo-assets/Dockerfile +++ b/hooli-demo-assets/Dockerfile @@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app ADD . . -RUN pip install -e . \ No newline at end of file +RUN python -m pip install -U uv +RUN uv pip sync requirements.txt --system \ No newline at end of file diff --git a/hooli-demo-assets/requirements.txt b/hooli-demo-assets/requirements.txt new file mode 100644 index 00000000..21435e8f --- /dev/null +++ b/hooli-demo-assets/requirements.txt @@ -0,0 +1,223 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=hooli-demo-assets/requirements.txt hooli-demo-assets/setup.py --system +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +astunparse==1.6.3 + # via dlt +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # dagster + # dlt + # typer +coloredlogs==14.0 + # via dagster +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via pyjwt +dagster==1.7.2 + # via + # dagster-cloud + # dagster-embedded-elt +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-embedded-elt==0.23.2 +dagster-pipes==1.7.2 + # via dagster +dlt==0.4.8 + # via dagster-embedded-elt +docstring-parser==0.16 + # via dagster +fsspec==2024.3.1 + # via + # dlt + # universal-pathlib +gitdb==4.0.11 + # via gitpython +github3-py==4.0.1 + # via dagster-cloud-cli +gitpython==3.1.43 + # via dlt +giturlparse==0.12.0 + # via dlt +greenlet==3.0.3 + # via sqlalchemy +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +hexbytes==1.2.0 + # via dlt +humanfriendly==10.0 + # via coloredlogs +humanize==4.9.0 + # via dlt +idna==3.7 + # via requests +jinja2==3.1.3 + # via dagster +jsonpath-ng==1.6.1 + # via dlt +makefun==1.15.2 + # via dlt +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako +mdurl==0.1.2 + # via markdown-it-py +orjson==3.9.10 + # via dlt +packaging==24.0 + # via + # dagster + # dagster-cloud-cli + # dlt +pathvalidate==3.2.0 + # via dlt +pendulum==3.0.0 + # via + # dagster + # dlt +pex==2.3.1 + # via dagster-cloud +ply==3.11 + # via jsonpath-ng +prompt-toolkit==3.0.36 + # via questionary +protobuf==4.25.3 + # via + # dagster + # grpcio-health-checking +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via dagster +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via rich +pyjwt==2.8.0 + # via github3-py +python-dateutil==2.9.0.post0 + # via + # croniter + # dagster + # github3-py + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +pytz==2024.1 + # via + # croniter + # dagster + # dlt +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli + # dlt +questionary==2.0.1 + # via + # dagster-cloud + # dagster-cloud-cli +requests==2.31.0 + # via + # dagster + # dagster-cloud + # dagster-cloud-cli + # dlt + # github3-py +requirements-parser==0.9.0 + # via dlt +rich==13.7.1 + # via + # dagster + # typer +semver==3.0.2 + # via dlt +setuptools==69.5.1 + # via + # dagster + # dlt +shellingham==1.5.4 + # via typer +simplejson==3.19.2 + # via dlt +six==1.16.0 + # via + # astunparse + # python-dateutil +sling==1.2.5 + # via dagster-embedded-elt +sling-linux-amd64==1.2.5 + # via sling +smmap==5.0.1 + # via gitdb +sqlalchemy==2.0.29 + # via + # alembic + # dagster +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via dagster +tenacity==8.2.3 + # via dlt +time-machine==2.14.1 + # via pendulum +tomli==2.0.1 + # via dagster +tomlkit==0.12.4 + # via dlt +toposort==1.10 + # via dagster +tqdm==4.66.2 + # via dagster +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli +types-setuptools==69.5.0.20240415 + # via requirements-parser +typing-extensions==4.11.0 + # via + # alembic + # dagster + # dlt + # pydantic + # pydantic-core + # sqlalchemy + # typer +tzdata==2024.1 + # via + # dlt + # pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==2.2.1 + # via requests +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit +wheel==0.43.0 + # via astunparse diff --git a/hooli_basics/Dockerfile b/hooli_basics/Dockerfile index 33751b97..6abd7f25 100644 --- a/hooli_basics/Dockerfile +++ b/hooli_basics/Dockerfile @@ -1,7 +1,8 @@ -FROM python:3.8-slim +FROM python:3.11-slim WORKDIR /opt/dagster/app ADD . . -RUN pip install -r requirements.txt +RUN python -m pip install -U uv +RUN uv pip sync requirements.txt --system diff --git a/hooli_basics/requirements.in b/hooli_basics/requirements.in new file mode 100644 index 00000000..cc02fb2c --- /dev/null +++ b/hooli_basics/requirements.in @@ -0,0 +1,5 @@ +pandas +html5lib +scikit-learn +dagster +dagster-cloud diff --git a/hooli_basics/requirements.txt b/hooli_basics/requirements.txt index cc02fb2c..1fe081d2 100644 --- a/hooli_basics/requirements.txt +++ b/hooli_basics/requirements.txt @@ -1,5 +1,182 @@ -pandas -html5lib -scikit-learn -dagster -dagster-cloud +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=hooli_basics/requirements.txt hooli_basics/requirements.in --system +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # dagster + # typer +coloredlogs==14.0 + # via dagster +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via pyjwt +dagster==1.7.2 + # via dagster-cloud +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-pipes==1.7.2 + # via dagster +docstring-parser==0.16 + # via dagster +fsspec==2024.3.1 + # via universal-pathlib +github3-py==4.0.1 + # via dagster-cloud-cli +greenlet==3.0.3 + # via sqlalchemy +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +html5lib==1.1 +humanfriendly==10.0 + # via coloredlogs +idna==3.7 + # via requests +jinja2==3.1.3 + # via dagster +joblib==1.4.0 + # via scikit-learn +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako +mdurl==0.1.2 + # via markdown-it-py +numpy==1.26.4 + # via + # pandas + # scikit-learn + # scipy +packaging==24.0 + # via + # dagster + # dagster-cloud-cli +pandas==2.2.2 +pendulum==3.0.0 + # via dagster +pex==2.3.1 + # via dagster-cloud +prompt-toolkit==3.0.36 + # via questionary +protobuf==4.25.3 + # via + # dagster + # grpcio-health-checking +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via dagster +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via rich +pyjwt==2.8.0 + # via github3-py +python-dateutil==2.9.0.post0 + # via + # croniter + # dagster + # github3-py + # pandas + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +pytz==2024.1 + # via + # croniter + # dagster + # pandas +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli +questionary==2.0.1 + # via + # dagster-cloud + # dagster-cloud-cli +requests==2.31.0 + # via + # dagster + # dagster-cloud + # dagster-cloud-cli + # github3-py +rich==13.7.1 + # via + # dagster + # typer +scikit-learn==1.4.2 +scipy==1.13.0 + # via scikit-learn +setuptools==69.5.1 + # via dagster +shellingham==1.5.4 + # via typer +six==1.16.0 + # via + # html5lib + # python-dateutil +sqlalchemy==2.0.29 + # via + # alembic + # dagster +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via dagster +threadpoolctl==3.4.0 + # via scikit-learn +time-machine==2.14.1 + # via pendulum +tomli==2.0.1 + # via dagster +toposort==1.10 + # via dagster +tqdm==4.66.2 + # via dagster +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli +typing-extensions==4.11.0 + # via + # alembic + # dagster + # pydantic + # pydantic-core + # sqlalchemy + # typer +tzdata==2024.1 + # via + # pandas + # pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==2.2.1 + # via requests +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit +webencodings==0.5.1 + # via html5lib diff --git a/hooli_batch_enrichment/Dockerfile b/hooli_batch_enrichment/Dockerfile index a401d314..4f6238db 100644 --- a/hooli_batch_enrichment/Dockerfile +++ b/hooli_batch_enrichment/Dockerfile @@ -4,4 +4,5 @@ WORKDIR /opt/dagster/app ADD . . -RUN pip install -e . +RUN python -m pip install -U uv +RUN uv pip sync requirements.txt --system diff --git a/hooli_batch_enrichment/requirements.txt b/hooli_batch_enrichment/requirements.txt new file mode 100644 index 00000000..6580e026 --- /dev/null +++ b/hooli_batch_enrichment/requirements.txt @@ -0,0 +1,177 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=hooli_batch_enrichment/requirements.txt hooli_batch_enrichment/setup.py --system +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # dagster + # typer +coloredlogs==14.0 + # via dagster +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via pyjwt +dagster==1.7.2 + # via + # dagster-cloud + # dagster-duckdb +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-duckdb==0.23.2 +dagster-pipes==1.7.2 + # via dagster +docstring-parser==0.16 + # via dagster +duckdb==0.10.2 + # via dagster-duckdb +fsspec==2024.3.1 + # via universal-pathlib +github3-py==4.0.1 + # via dagster-cloud-cli +greenlet==3.0.3 + # via sqlalchemy +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +humanfriendly==10.0 + # via coloredlogs +idna==3.7 + # via requests +jinja2==3.1.3 + # via dagster +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako +mdurl==0.1.2 + # via markdown-it-py +numpy==1.26.4 + # via pandas +packaging==24.0 + # via + # dagster + # dagster-cloud-cli +pandas==2.2.2 +pendulum==3.0.0 + # via dagster +pex==2.3.1 + # via dagster-cloud +prompt-toolkit==3.0.36 + # via questionary +protobuf==4.25.3 + # via + # dagster + # grpcio-health-checking +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via dagster +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via rich +pyjwt==2.8.0 + # via github3-py +python-dateutil==2.9.0.post0 + # via + # croniter + # dagster + # github3-py + # pandas + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +pytz==2024.1 + # via + # croniter + # dagster + # pandas +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli + # responses +questionary==2.0.1 + # via + # dagster-cloud + # dagster-cloud-cli +requests==2.31.0 + # via + # dagster + # dagster-cloud + # dagster-cloud-cli + # github3-py + # responses +responses==0.25.0 +rich==13.7.1 + # via + # dagster + # typer +setuptools==69.5.1 + # via dagster +shellingham==1.5.4 + # via typer +six==1.16.0 + # via python-dateutil +sqlalchemy==2.0.29 + # via + # alembic + # dagster +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via dagster +time-machine==2.14.1 + # via pendulum +tomli==2.0.1 + # via dagster +toposort==1.10 + # via dagster +tqdm==4.66.2 + # via dagster +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli +typing-extensions==4.11.0 + # via + # alembic + # dagster + # pydantic + # pydantic-core + # sqlalchemy + # typer +tzdata==2024.1 + # via + # pandas + # pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==2.2.1 + # via + # requests + # responses +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit diff --git a/hooli_snowflake_insights/Dockerfile b/hooli_snowflake_insights/Dockerfile index 4ac0381f..5f1b82de 100644 --- a/hooli_snowflake_insights/Dockerfile +++ b/hooli_snowflake_insights/Dockerfile @@ -9,4 +9,5 @@ RUN python -m pip install git+https://github.com/wbond/oscrypto.git@d5f3437ed242 ADD . . -RUN pip install -r requirements.txt +RUN python -m pip install -U uv +RUN uv pip sync requirements.txt --system diff --git a/hooli_snowflake_insights/requirements.in b/hooli_snowflake_insights/requirements.in new file mode 100644 index 00000000..57b6f6e4 --- /dev/null +++ b/hooli_snowflake_insights/requirements.in @@ -0,0 +1,6 @@ +dagster +dagster-cloud +dagster-dbt +dagster-snowflake +gql +requests_toolbelt \ No newline at end of file diff --git a/hooli_snowflake_insights/requirements.txt b/hooli_snowflake_insights/requirements.txt index 57b6f6e4..8940098f 100644 --- a/hooli_snowflake_insights/requirements.txt +++ b/hooli_snowflake_insights/requirements.txt @@ -1,6 +1,318 @@ -dagster -dagster-cloud -dagster-dbt -dagster-snowflake -gql -requests_toolbelt \ No newline at end of file +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=hooli_snowflake_insights/requirements.txt hooli_snowflake_insights/requirements.in --system +agate==1.7.1 + # via dbt-core +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via gql +asn1crypto==1.5.1 + # via snowflake-connector-python +attrs==23.2.0 + # via + # jsonschema + # referencing +babel==2.14.0 + # via agate +backoff==2.2.1 + # via gql +certifi==2024.2.2 + # via + # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # cryptography + # dbt-core + # snowflake-connector-python +charset-normalizer==3.3.2 + # via + # requests + # snowflake-connector-python +click==8.1.7 + # via + # dagster + # dbt-core + # dbt-semantic-interfaces + # typer +colorama==0.4.6 + # via dbt-core +coloredlogs==14.0 + # via dagster +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via + # pyjwt + # pyopenssl + # snowflake-connector-python +dagster==1.7.2 + # via + # dagster-cloud + # dagster-dbt + # dagster-snowflake +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-dbt==0.23.2 +dagster-pipes==1.7.2 + # via dagster +dagster-snowflake==0.23.2 +dbt-core==1.7.13 + # via dagster-dbt +dbt-extractor==0.5.1 + # via dbt-core +dbt-semantic-interfaces==0.4.4 + # via dbt-core +docstring-parser==0.16 + # via dagster +exceptiongroup==1.2.1 + # via anyio +filelock==3.13.4 + # via snowflake-connector-python +fsspec==2024.3.1 + # via universal-pathlib +github3-py==4.0.1 + # via dagster-cloud-cli +gql==3.5.0 +graphql-core==3.2.3 + # via gql +greenlet==3.0.3 + # via sqlalchemy +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +humanfriendly==10.0 + # via coloredlogs +idna==3.7 + # via + # anyio + # dbt-core + # requests + # snowflake-connector-python + # yarl +importlib-metadata==6.11.0 + # via dbt-semantic-interfaces +isodate==0.6.1 + # via + # agate + # dbt-core +jinja2==3.1.3 + # via + # dagster + # dagster-dbt + # dbt-core + # dbt-semantic-interfaces +jsonschema==4.21.1 + # via + # dbt-core + # dbt-semantic-interfaces +jsonschema-specifications==2023.12.1 + # via jsonschema +leather==0.4.0 + # via agate +logbook==1.5.3 + # via dbt-core +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako +mashumaro==3.12 + # via dbt-core +mdurl==0.1.2 + # via markdown-it-py +minimal-snowplow-tracker==0.0.2 + # via dbt-core +more-itertools==10.2.0 + # via dbt-semantic-interfaces +msgpack==1.0.8 + # via mashumaro +multidict==6.0.5 + # via yarl +networkx==3.3 + # via + # dagster-dbt + # dbt-core +orjson==3.10.1 + # via dagster-dbt +packaging==24.0 + # via + # dagster + # dagster-cloud-cli + # dagster-dbt + # dbt-core + # snowflake-connector-python +parsedatetime==2.6 + # via agate +pathspec==0.11.2 + # via dbt-core +pendulum==3.0.0 + # via dagster +pex==2.3.1 + # via dagster-cloud +platformdirs==4.2.0 + # via snowflake-connector-python +prompt-toolkit==3.0.36 + # via questionary +protobuf==4.25.3 + # via + # dagster + # dbt-core + # grpcio-health-checking +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via + # dagster + # dbt-semantic-interfaces +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via rich +pyjwt==2.8.0 + # via + # github3-py + # snowflake-connector-python +pyopenssl==24.1.0 + # via snowflake-connector-python +python-dateutil==2.9.0.post0 + # via + # croniter + # dagster + # dbt-semantic-interfaces + # github3-py + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +python-slugify==8.0.4 + # via agate +pytimeparse==1.1.8 + # via agate +pytz==2024.1 + # via + # croniter + # dagster + # dbt-core + # snowflake-connector-python +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli + # dbt-core + # dbt-semantic-interfaces +questionary==2.0.1 + # via + # dagster-cloud + # dagster-cloud-cli +referencing==0.34.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via + # dagster + # dagster-cloud + # dagster-cloud-cli + # dagster-dbt + # dbt-core + # github3-py + # minimal-snowplow-tracker + # requests-toolbelt + # snowflake-connector-python +requests-toolbelt==1.0.0 +rich==13.7.1 + # via + # dagster + # dagster-dbt + # typer +rpds-py==0.18.0 + # via + # jsonschema + # referencing +setuptools==69.5.1 + # via dagster +shellingham==1.5.4 + # via typer +six==1.16.0 + # via + # isodate + # minimal-snowplow-tracker + # python-dateutil +sniffio==1.3.1 + # via anyio +snowflake-connector-python==3.9.0 + # via dagster-snowflake +sortedcontainers==2.4.0 + # via snowflake-connector-python +sqlalchemy==2.0.29 + # via + # alembic + # dagster +sqlglot==23.10.0 + # via dagster-dbt +sqlglotrs==0.2.0 + # via sqlglot +sqlparse==0.5.0 + # via dbt-core +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via dagster +text-unidecode==1.3 + # via python-slugify +time-machine==2.14.1 + # via pendulum +tomli==2.0.1 + # via dagster +tomlkit==0.12.4 + # via snowflake-connector-python +toposort==1.10 + # via dagster +tqdm==4.66.2 + # via dagster +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli + # dagster-dbt +typing-extensions==4.11.0 + # via + # alembic + # anyio + # dagster + # dbt-core + # dbt-semantic-interfaces + # mashumaro + # pydantic + # pydantic-core + # snowflake-connector-python + # sqlalchemy + # typer +tzdata==2024.1 + # via pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==1.26.18 + # via + # dbt-core + # requests +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit +yarl==1.9.4 + # via gql +zipp==3.18.1 + # via importlib-metadata diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..01467b52 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,661 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=requirements.txt setup.py --system +agate==1.7.1 + # via + # dbt-core + # dbt-snowflake +alembic==1.13.1 + # via dagster +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via gql +asn1crypto==1.5.1 + # via snowflake-connector-python +asttokens==2.4.1 + # via stack-data +attrs==23.2.0 + # via + # jsonschema + # referencing +babel==2.14.0 + # via agate +backoff==2.2.1 + # via gql +beautifulsoup4==4.12.3 + # via nbconvert +bleach==6.1.0 + # via nbconvert +boto3==1.34.87 + # via dagster-aws +botocore==1.34.87 + # via + # boto3 + # s3transfer +cachetools==5.3.3 + # via google-auth +certifi==2024.2.2 + # via + # kubernetes + # requests + # snowflake-connector-python +cffi==1.16.0 + # via + # cryptography + # dbt-core + # snowflake-connector-python +charset-normalizer==3.3.2 + # via + # requests + # snowflake-connector-python +click==8.1.7 + # via + # dagster + # databricks-cli + # dbt-core + # dbt-semantic-interfaces + # papermill + # typer +colorama==0.4.6 + # via dbt-core +coloredlogs==14.0 + # via dagster +comm==0.2.2 + # via ipykernel +contourpy==1.2.1 + # via matplotlib +croniter==2.0.3 + # via dagster +cryptography==42.0.5 + # via + # pyjwt + # pyopenssl + # secretstorage + # snowflake-connector-python +cycler==0.12.1 + # via matplotlib +dagster==1.7.2 + # via + # dagster-aws + # dagster-cloud + # dagster-databricks + # dagster-dbt + # dagster-duckdb + # dagster-duckdb-pandas + # dagster-k8s + # dagster-pyspark + # dagster-snowflake + # dagster-snowflake-pandas + # dagster-spark + # dagstermill +dagster-aws==0.23.2 +dagster-cloud==1.7.2 +dagster-cloud-cli==1.7.2 + # via dagster-cloud +dagster-databricks==0.23.2 +dagster-dbt==0.23.2 +dagster-duckdb==0.23.2 + # via dagster-duckdb-pandas +dagster-duckdb-pandas==0.23.2 +dagster-k8s==0.23.2 +dagster-pipes==1.7.2 + # via + # dagster + # dagster-databricks +dagster-pyspark==0.23.2 + # via dagster-databricks +dagster-snowflake==0.23.2 + # via dagster-snowflake-pandas +dagster-snowflake-pandas==0.23.2 +dagster-spark==0.23.2 + # via dagster-pyspark +dagstermill==0.23.2 +databricks-api==0.9.0 + # via dagster-databricks +databricks-cli==0.18.0 + # via + # dagster-databricks + # databricks-api +databricks-sdk==0.8.0 + # via dagster-databricks +dbt-core==1.7.13 + # via + # dagster-dbt + # dbt-duckdb + # dbt-snowflake +dbt-duckdb==1.7.4 +dbt-extractor==0.5.1 + # via dbt-core +dbt-semantic-interfaces==0.4.4 + # via dbt-core +dbt-snowflake==1.7.3 +debugpy==1.8.1 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +docstring-parser==0.16 + # via dagster +duckdb==0.10.2 + # via + # dagster-duckdb + # dbt-duckdb +entrypoints==0.4 + # via + # jupyter-client + # papermill +exceptiongroup==1.2.1 + # via + # anyio + # ipython +executing==2.0.1 + # via stack-data +fastjsonschema==2.19.1 + # via nbformat +filelock==3.13.4 + # via snowflake-connector-python +fonttools==4.51.0 + # via matplotlib +fsspec==2024.3.1 + # via universal-pathlib +github3-py==4.0.1 + # via dagster-cloud-cli +google-auth==2.29.0 + # via + # dagster-k8s + # kubernetes +gql==3.5.0 +graphql-core==3.2.3 + # via gql +greenlet==3.0.3 + # via sqlalchemy +grpcio==1.62.2 + # via + # dagster + # grpcio-health-checking +grpcio-health-checking==1.62.2 + # via dagster +html5lib==1.1 +humanfriendly==10.0 + # via coloredlogs +idna==3.7 + # via + # anyio + # dbt-core + # requests + # snowflake-connector-python + # yarl +importlib-metadata==6.11.0 + # via + # dbt-semantic-interfaces + # keyring +ipykernel==6.29.4 + # via dagstermill +ipython==8.23.0 + # via + # ipykernel + # scrapbook +ipython-genutils==0.2.0 + # via dagstermill +isodate==0.6.1 + # via + # agate + # dbt-core +jaraco-classes==3.4.0 + # via keyring +jedi==0.19.1 + # via ipython +jeepney==0.8.0 + # via + # keyring + # secretstorage +jinja2==3.1.3 + # via + # dagster + # dagster-dbt + # dbt-core + # dbt-semantic-interfaces + # nbconvert +jmespath==1.0.1 + # via + # boto3 + # botocore +joblib==1.4.0 + # via scikit-learn +jsonschema==4.21.1 + # via + # dbt-core + # dbt-semantic-interfaces + # nbformat + # scrapbook +jsonschema-specifications==2023.12.1 + # via jsonschema +jupyter-client==7.4.9 + # via + # dagstermill + # ipykernel + # nbclient +jupyter-core==5.7.2 + # via + # ipykernel + # jupyter-client + # nbclient + # nbconvert + # nbformat +jupyterlab-pygments==0.3.0 + # via nbconvert +keyring==24.3.1 + # via snowflake-connector-python +kiwisolver==1.4.5 + # via matplotlib +kubernetes==29.0.0 + # via dagster-k8s +leather==0.4.0 + # via agate +logbook==1.5.3 + # via dbt-core +mako==1.3.3 + # via alembic +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # mako + # nbconvert +mashumaro==3.12 + # via dbt-core +matplotlib==3.8.4 + # via + # mizani + # plotnine +matplotlib-inline==0.1.7 + # via + # ipykernel + # ipython +mdurl==0.1.2 + # via markdown-it-py +minimal-snowplow-tracker==0.0.2 + # via dbt-core +mistune==3.0.2 + # via nbconvert +mizani==0.9.3 + # via plotnine +more-itertools==10.2.0 + # via + # dbt-semantic-interfaces + # jaraco-classes +msgpack==1.0.8 + # via mashumaro +multidict==6.0.5 + # via yarl +nbclient==0.10.0 + # via + # nbconvert + # papermill +nbconvert==7.16.3 + # via dagstermill +nbformat==5.10.4 + # via + # nbclient + # nbconvert + # papermill +nest-asyncio==1.6.0 + # via + # ipykernel + # jupyter-client +networkx==3.3 + # via + # dagster-dbt + # dbt-core +numpy==1.26.4 + # via + # contourpy + # matplotlib + # mizani + # pandas + # patsy + # plotnine + # pyarrow + # scikit-learn + # scipy + # statsmodels +oauthlib==3.2.2 + # via + # databricks-cli + # kubernetes + # requests-oauthlib +orjson==3.10.1 + # via dagster-dbt +packaging==24.0 + # via + # dagster + # dagster-aws + # dagster-cloud-cli + # dagster-dbt + # dagstermill + # dbt-core + # ipykernel + # matplotlib + # nbconvert + # snowflake-connector-python + # statsmodels +pandas==2.0.3 + # via + # dagster-duckdb-pandas + # dagster-snowflake-pandas + # mizani + # plotnine + # scrapbook + # snowflake-connector-python + # statsmodels +pandocfilters==1.5.1 + # via nbconvert +papermill==2.5.0 + # via + # dagstermill + # scrapbook +parsedatetime==2.6 + # via agate +parso==0.8.4 + # via jedi +pathspec==0.11.2 + # via dbt-core +patsy==0.5.6 + # via + # plotnine + # statsmodels +pendulum==3.0.0 + # via dagster +pex==2.3.1 + # via dagster-cloud +pexpect==4.9.0 + # via ipython +pillow==10.3.0 + # via matplotlib +platformdirs==4.2.0 + # via + # jupyter-core + # snowflake-connector-python +plotnine==0.12.4 +prompt-toolkit==3.0.43 + # via + # ipython + # questionary +protobuf==4.25.3 + # via + # dagster + # dbt-core + # grpcio-health-checking +psutil==5.9.8 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +py4j==0.10.9.7 + # via pyspark +pyarrow==15.0.2 + # via + # scrapbook + # snowflake-connector-python +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pycparser==2.22 + # via cffi +pydantic==2.7.0 + # via + # dagster + # dbt-semantic-interfaces +pydantic-core==2.18.1 + # via pydantic +pygments==2.17.2 + # via + # ipython + # nbconvert + # rich +pyjwt==2.8.0 + # via + # databricks-cli + # github3-py + # snowflake-connector-python +pyopenssl==24.1.0 + # via snowflake-connector-python +pyparsing==3.1.2 + # via matplotlib +pyspark==3.5.1 + # via dagster-pyspark +python-dateutil==2.9.0.post0 + # via + # botocore + # croniter + # dagster + # dbt-semantic-interfaces + # github3-py + # jupyter-client + # kubernetes + # matplotlib + # pandas + # pendulum + # time-machine +python-dotenv==1.0.1 + # via dagster +python-slugify==8.0.4 + # via agate +pytimeparse==1.1.8 + # via agate +pytz==2024.1 + # via + # croniter + # dagster + # dbt-core + # pandas + # snowflake-connector-python +pyyaml==6.0.1 + # via + # dagster + # dagster-cloud-cli + # dbt-core + # dbt-semantic-interfaces + # kubernetes + # papermill + # responses +pyzmq==26.0.1 + # via + # ipykernel + # jupyter-client +questionary==1.10.0 + # via + # dagster-cloud + # dagster-cloud-cli +referencing==0.34.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via + # dagster + # dagster-aws + # dagster-cloud + # dagster-cloud-cli + # dagster-dbt + # dagster-snowflake-pandas + # databricks-cli + # databricks-sdk + # dbt-core + # github3-py + # kubernetes + # minimal-snowplow-tracker + # papermill + # requests-oauthlib + # requests-toolbelt + # responses + # snowflake-connector-python +requests-oauthlib==2.0.0 + # via kubernetes +requests-toolbelt==1.0.0 +responses==0.25.0 +rich==13.7.1 + # via + # dagster + # dagster-dbt + # typer +rpds-py==0.18.0 + # via + # jsonschema + # referencing +rsa==4.9 + # via google-auth +s3transfer==0.10.1 + # via boto3 +scikit-learn==1.4.2 +scipy==1.13.0 + # via + # mizani + # plotnine + # scikit-learn + # statsmodels +scrapbook==0.5.0 + # via dagstermill +secretstorage==3.3.3 + # via keyring +setuptools==69.5.1 + # via dagster +shellingham==1.5.4 + # via typer +six==1.16.0 + # via + # asttokens + # bleach + # databricks-cli + # html5lib + # isodate + # kubernetes + # minimal-snowplow-tracker + # patsy + # python-dateutil +sniffio==1.3.1 + # via anyio +snowflake-connector-python==3.9.0 + # via + # dagster-snowflake + # dagster-snowflake-pandas + # dbt-snowflake + # snowflake-sqlalchemy +snowflake-sqlalchemy==1.5.3 + # via dagster-snowflake-pandas +sortedcontainers==2.4.0 + # via snowflake-connector-python +soupsieve==2.5 + # via beautifulsoup4 +sqlalchemy==1.4.52 + # via + # alembic + # dagster + # dagster-snowflake-pandas + # snowflake-sqlalchemy +sqlglot==23.10.0 + # via dagster-dbt +sqlglotrs==0.2.0 + # via sqlglot +sqlparse==0.5.0 + # via dbt-core +stack-data==0.6.3 + # via ipython +statsmodels==0.14.2 + # via plotnine +structlog==24.1.0 + # via dagster +tabulate==0.9.0 + # via + # dagster + # databricks-cli +tenacity==8.2.3 + # via papermill +text-unidecode==1.3 + # via python-slugify +threadpoolctl==3.4.0 + # via scikit-learn +time-machine==2.14.1 + # via pendulum +tinycss2==1.2.1 + # via nbconvert +tomli==2.0.1 + # via dagster +tomlkit==0.12.4 + # via snowflake-connector-python +toposort==1.10 + # via dagster +tornado==6.4 + # via + # ipykernel + # jupyter-client +tqdm==4.66.2 + # via + # dagster + # papermill +traitlets==5.14.3 + # via + # comm + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat +typer==0.12.3 + # via + # dagster-cloud + # dagster-cloud-cli + # dagster-dbt +typing-extensions==4.11.0 + # via + # alembic + # anyio + # dagster + # dbt-core + # dbt-semantic-interfaces + # ipython + # mashumaro + # pydantic + # pydantic-core + # snowflake-connector-python + # typer +tzdata==2024.1 + # via + # pandas + # pendulum +universal-pathlib==0.2.2 + # via dagster +uritemplate==4.1.1 + # via github3-py +urllib3==1.26.18 + # via + # botocore + # databricks-cli + # dbt-core + # kubernetes + # requests + # responses +watchdog==4.0.0 + # via dagster +wcwidth==0.2.13 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # html5lib + # tinycss2 +websocket-client==1.7.0 + # via kubernetes +yarl==1.9.4 + # via gql +zipp==3.18.1 + # via importlib-metadata