From d93fbba81452fd5e950d8038ad5346a7cc0a0048 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 20:01:40 -0400 Subject: [PATCH 01/41] remove vscode and ds_store from git tracking --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 094ee4a9..b8d4accc 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,9 @@ cython_debug/ # testing artifacts /logs + +# MacOS +.DS_Store + +# vscode +.vscode/ From 97f7ef756b1ceb7dfc04c616ce9e67abab6ee728 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 20:03:17 -0400 Subject: [PATCH 02/41] default to psycopg2-binary everywhere --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a99829d9..3110e201 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - # install `psycopg2` on linux (assumed production) - 'psycopg2>=2.9,<3.0; platform_system == "Linux"', - # install `psycopg2-binary` on macos/windows (assumed development) - 'psycopg2-binary>=2.9,<3.0; platform_system != "Linux"', + "psycopg2-binary>=2.9,<3.0", "dbt-adapters>=0.1.0a1,<2.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0a1", @@ -46,6 +43,9 @@ Changelog = "https://github.com/dbt-labs/dbt-postgres/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.build.hooks.custom] +path = "./hatch_hooks.py" + [tool.hatch.build.targets.sdist] include = ["dbt"] From 54895ebed4587fe8d4ca60266516c7c064b7fb06 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 20:04:05 -0400 Subject: [PATCH 03/41] add post build hook to swap out psycopg2-binary for psycopg2 based on DBT_PSYCOPG2_NAME --- hatch_hooks.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 hatch_hooks.py diff --git a/hatch_hooks.py b/hatch_hooks.py new file mode 100644 index 00000000..fc036128 --- /dev/null +++ b/hatch_hooks.py @@ -0,0 +1,35 @@ +import os +import subprocess +import sys + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.plugin import hookimpl + + +class Psycopg2NoBinary(BuildHookInterface): + """ + Custom build hook to install psycopg2 instead of psycopg2-binary based on the presence of `DBT_PSYCOPG2_NAME`. + This is necessary as psycopg2-binary is better for local development, but psycopg2 is better for production. + """ + + PLUGIN_NAME = "psycopg2" + + def finalize(self, version, build_data, artifact_path) -> None: + if os.getenv("DBT_PSYCOPG2_NAME", "") == "psycopg2": + psycopg2_binary_pinned = [ + package + for package in build_data["dependencies"] + if package.startswith("psycopg2-binary") + ].pop() + psycopg2_pinned = psycopg2_binary_pinned.replace("-binary", "") + subprocess.check_call( + [sys.executable, "-m", "pip", "-y", "uninstall", "psycopg2-binary"] + ) + subprocess.check_call( + [sys.executable, "-m", "pip", "-y", "install", f'"{psycopg2_pinned}"'] + ) + + +@hookimpl +def hatch_register_build_hook(): + return Psycopg2NoBinary From cb7307bc31ca06001603b19e4af77b5363623991 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 20:18:30 -0400 Subject: [PATCH 04/41] update integration tests for the new psycopg2 install approach --- .github/scripts/check_psycopg2_install.sh | 10 +++++++ .github/workflows/integration-tests.yml | 34 +++++++++++++---------- 2 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 .github/scripts/check_psycopg2_install.sh diff --git a/.github/scripts/check_psycopg2_install.sh b/.github/scripts/check_psycopg2_install.sh new file mode 100644 index 00000000..5d96a16e --- /dev/null +++ b/.github/scripts/check_psycopg2_install.sh @@ -0,0 +1,10 @@ +python -m pip install . +PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) + +echo $PSYCOPG2_PIP_ENTRY +PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" +echo $PSYCOPG2_NAME + +if [[ "${PSYCOPG2_NAME}" != "${PSYCOPG2_EXPECTED_NAME}" ]]; then + exit 1 +fi diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 1aefb7f5..54574a80 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -102,24 +102,30 @@ jobs: psycopg2-check: name: "Test psycopg2 build version" - runs-on: ${{ matrix.scenario.platform }} + runs-on: ${{ matrix.platform }} strategy: fail-fast: false matrix: - scenario: - - {platform: ubuntu-latest, psycopg2-name: psycopg2} - - {platform: macos-12, psycopg2-name: psycopg2-binary} + platform: + - ubuntu-24.04 + - macos-12 steps: - name: "Check out repository" uses: actions/checkout@v4 - - name: "Test psycopg2 name" - run: | - python -m pip install . - PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) - echo $PSYCOPG2_PIP_ENTRY - PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" - echo $PSYCOPG2_NAME - if [[ "${PSYCOPG2_NAME}" != "${{ matrix.scenario.psycopg2-name }}" ]]; then - exit 1 - fi + - name: "Test psycopg2 name - default" + run: ./.github/scripts/check_psycopg2_install.sh + env: + PSYCOPG2_EXPECTED_NAME: psycopg2-binary + + - name: "Test psycopg2 name - override" + run: ./.github/scripts/check_psycopg2_install.sh + env: + DBT_PSYCOPG2_NAME: psycopg2 + PSYCOPG2_EXPECTED_NAME: psycopg2 + + - name: "Test psycopg2 name - invalide override" + run: ./.github/scripts/check_psycopg2_install.sh + env: + DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers + PSYCOPG2_EXPECTED_NAME: psycopg2-binary From 94da8504b704798eb3eab1691eec24e97d051a40 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 20:26:22 -0400 Subject: [PATCH 05/41] changelog --- .changes/unreleased/Fixes-20240605-202614.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240605-202614.yaml diff --git a/.changes/unreleased/Fixes-20240605-202614.yaml b/.changes/unreleased/Fixes-20240605-202614.yaml new file mode 100644 index 00000000..b7ab8eb0 --- /dev/null +++ b/.changes/unreleased/Fixes-20240605-202614.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Default to psycopg2-binary and allow overriding to psycopg2 via DBT_PSYCOPG2_NAME + (restores previous behavior) +time: 2024-06-05T20:26:14.801254-04:00 +custom: + Author: mikealfare + Issue: "96" From 19e5a5289a8f1352310bf2bdebc0a52acf9fb5ea Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 20:53:17 -0400 Subject: [PATCH 06/41] update post-build hook to a pre-build hook --- hatch_hooks.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/hatch_hooks.py b/hatch_hooks.py index fc036128..ae9c4f15 100644 --- a/hatch_hooks.py +++ b/hatch_hooks.py @@ -1,6 +1,4 @@ import os -import subprocess -import sys from hatchling.builders.hooks.plugin.interface import BuildHookInterface from hatchling.plugin import hookimpl @@ -14,7 +12,7 @@ class Psycopg2NoBinary(BuildHookInterface): PLUGIN_NAME = "psycopg2" - def finalize(self, version, build_data, artifact_path) -> None: + def initialize(self, version, build_data) -> None: if os.getenv("DBT_PSYCOPG2_NAME", "") == "psycopg2": psycopg2_binary_pinned = [ package @@ -22,12 +20,7 @@ def finalize(self, version, build_data, artifact_path) -> None: if package.startswith("psycopg2-binary") ].pop() psycopg2_pinned = psycopg2_binary_pinned.replace("-binary", "") - subprocess.check_call( - [sys.executable, "-m", "pip", "-y", "uninstall", "psycopg2-binary"] - ) - subprocess.check_call( - [sys.executable, "-m", "pip", "-y", "install", f'"{psycopg2_pinned}"'] - ) + build_data["dependencies"].append(psycopg2_pinned) @hookimpl From f86d9d5169ba5b634182d569e4a395eee87db876 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 20:58:13 -0400 Subject: [PATCH 07/41] replace script with inline execution --- .github/scripts/check_psycopg2_install.sh | 10 ------- .github/workflows/integration-tests.yml | 36 +++++++++++++++++++++-- 2 files changed, 33 insertions(+), 13 deletions(-) delete mode 100644 .github/scripts/check_psycopg2_install.sh diff --git a/.github/scripts/check_psycopg2_install.sh b/.github/scripts/check_psycopg2_install.sh deleted file mode 100644 index 5d96a16e..00000000 --- a/.github/scripts/check_psycopg2_install.sh +++ /dev/null @@ -1,10 +0,0 @@ -python -m pip install . -PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) - -echo $PSYCOPG2_PIP_ENTRY -PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" -echo $PSYCOPG2_NAME - -if [[ "${PSYCOPG2_NAME}" != "${PSYCOPG2_EXPECTED_NAME}" ]]; then - exit 1 -fi diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 54574a80..fff13886 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -114,18 +114,48 @@ jobs: uses: actions/checkout@v4 - name: "Test psycopg2 name - default" - run: ./.github/scripts/check_psycopg2_install.sh + run: | + python -m pip install . + PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) + + echo $PSYCOPG2_PIP_ENTRY + PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" + echo $PSYCOPG2_NAME + + if [[ "${PSYCOPG2_NAME}" != "${PSYCOPG2_EXPECTED_NAME}" ]]; then + exit 1 + fi env: PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 name - override" - run: ./.github/scripts/check_psycopg2_install.sh + run: | + python -m pip install . + PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) + + echo $PSYCOPG2_PIP_ENTRY + PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" + echo $PSYCOPG2_NAME + + if [[ "${PSYCOPG2_NAME}" != "${PSYCOPG2_EXPECTED_NAME}" ]]; then + exit 1 + fi env: DBT_PSYCOPG2_NAME: psycopg2 PSYCOPG2_EXPECTED_NAME: psycopg2 - name: "Test psycopg2 name - invalide override" - run: ./.github/scripts/check_psycopg2_install.sh + run: | + python -m pip install . + PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) + + echo $PSYCOPG2_PIP_ENTRY + PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" + echo $PSYCOPG2_NAME + + if [[ "${PSYCOPG2_NAME}" != "${PSYCOPG2_EXPECTED_NAME}" ]]; then + exit 1 + fi env: DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers PSYCOPG2_EXPECTED_NAME: psycopg2-binary From 9854f22f55244bc55ea964b6c5c980844016550c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 5 Jun 2024 23:18:01 -0400 Subject: [PATCH 08/41] use hatch run instead of pip install --- .github/workflows/integration-tests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index fff13886..9fe54169 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -113,10 +113,12 @@ jobs: - name: "Check out repository" uses: actions/checkout@v4 + - name: Setup `hatch` + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + - name: "Test psycopg2 name - default" run: | - python -m pip install . - PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) + PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) echo $PSYCOPG2_PIP_ENTRY PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" @@ -128,10 +130,9 @@ jobs: env: PSYCOPG2_EXPECTED_NAME: psycopg2-binary - - name: "Test psycopg2 name - override" + - name: "Test psycopg2 name - invalide override" run: | - python -m pip install . - PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) + PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) echo $PSYCOPG2_PIP_ENTRY PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" @@ -141,13 +142,12 @@ jobs: exit 1 fi env: - DBT_PSYCOPG2_NAME: psycopg2 - PSYCOPG2_EXPECTED_NAME: psycopg2 + DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers + PSYCOPG2_EXPECTED_NAME: psycopg2-binary - - name: "Test psycopg2 name - invalide override" + - name: "Test psycopg2 name - override" run: | - python -m pip install . - PSYCOPG2_PIP_ENTRY=$(pip list | grep "psycopg2 " || pip list | grep psycopg2-binary) + PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) echo $PSYCOPG2_PIP_ENTRY PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" @@ -157,5 +157,5 @@ jobs: exit 1 fi env: - DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers - PSYCOPG2_EXPECTED_NAME: psycopg2-binary + DBT_PSYCOPG2_NAME: psycopg2 + PSYCOPG2_EXPECTED_NAME: psycopg2 From 7504b56928de0af2ea937efae6dc91fa82a2a31a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jun 2024 00:35:01 -0400 Subject: [PATCH 09/41] typo in the env var --- hatch_hooks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hatch_hooks.py b/hatch_hooks.py index ae9c4f15..8f443d74 100644 --- a/hatch_hooks.py +++ b/hatch_hooks.py @@ -18,7 +18,8 @@ def initialize(self, version, build_data) -> None: package for package in build_data["dependencies"] if package.startswith("psycopg2-binary") - ].pop() + ] + build_data["depenndencies"].pop(psycopg2_binary_pinned) psycopg2_pinned = psycopg2_binary_pinned.replace("-binary", "") build_data["dependencies"].append(psycopg2_pinned) From 528d3ce9fa09bf133d16192d1318658c9514ad49 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jun 2024 00:53:08 -0400 Subject: [PATCH 10/41] fix a type --- hatch_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch_hooks.py b/hatch_hooks.py index 8f443d74..b7cd9c77 100644 --- a/hatch_hooks.py +++ b/hatch_hooks.py @@ -19,7 +19,7 @@ def initialize(self, version, build_data) -> None: for package in build_data["dependencies"] if package.startswith("psycopg2-binary") ] - build_data["depenndencies"].pop(psycopg2_binary_pinned) + build_data["dependencies"].pop(psycopg2_binary_pinned) psycopg2_pinned = psycopg2_binary_pinned.replace("-binary", "") build_data["dependencies"].append(psycopg2_pinned) From 07b0c226358d44fe3cb9284211ccd97966ef8092 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jun 2024 01:04:56 -0400 Subject: [PATCH 11/41] syntax --- .github/workflows/integration-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 9fe54169..f26f9f12 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -124,7 +124,7 @@ jobs: PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" echo $PSYCOPG2_NAME - if [[ "${PSYCOPG2_NAME}" != "${PSYCOPG2_EXPECTED_NAME}" ]]; then + if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi env: @@ -138,7 +138,7 @@ jobs: PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" echo $PSYCOPG2_NAME - if [[ "${PSYCOPG2_NAME}" != "${PSYCOPG2_EXPECTED_NAME}" ]]; then + if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi env: @@ -153,7 +153,7 @@ jobs: PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" echo $PSYCOPG2_NAME - if [[ "${PSYCOPG2_NAME}" != "${PSYCOPG2_EXPECTED_NAME}" ]]; then + if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi env: From dbfa07d73348ef66d4a59999c03a347b7e584e7c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jun 2024 01:32:23 -0400 Subject: [PATCH 12/41] typo --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index f26f9f12..3b9eb02a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -130,7 +130,7 @@ jobs: env: PSYCOPG2_EXPECTED_NAME: psycopg2-binary - - name: "Test psycopg2 name - invalide override" + - name: "Test psycopg2 name - invalid override" run: | PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) From 1e9722430abaa10e56d13ebea75d56e697481823 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jun 2024 01:32:35 -0400 Subject: [PATCH 13/41] get the value from the list --- hatch_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hatch_hooks.py b/hatch_hooks.py index b7cd9c77..95a78d70 100644 --- a/hatch_hooks.py +++ b/hatch_hooks.py @@ -18,7 +18,7 @@ def initialize(self, version, build_data) -> None: package for package in build_data["dependencies"] if package.startswith("psycopg2-binary") - ] + ].pop() build_data["dependencies"].pop(psycopg2_binary_pinned) psycopg2_pinned = psycopg2_binary_pinned.replace("-binary", "") build_data["dependencies"].append(psycopg2_pinned) From 8b1d3fd052bc57d6e054f76829a3c47a4b4800ea Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jun 2024 01:37:04 -0400 Subject: [PATCH 14/41] debugging --- hatch_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hatch_hooks.py b/hatch_hooks.py index 95a78d70..9a4cabef 100644 --- a/hatch_hooks.py +++ b/hatch_hooks.py @@ -13,6 +13,7 @@ class Psycopg2NoBinary(BuildHookInterface): PLUGIN_NAME = "psycopg2" def initialize(self, version, build_data) -> None: + print(build_data) if os.getenv("DBT_PSYCOPG2_NAME", "") == "psycopg2": psycopg2_binary_pinned = [ package From 5460c611b65c2a624c14f040de63f124be1afc76 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 6 Jun 2024 02:00:19 -0400 Subject: [PATCH 15/41] type --- .github/workflows/integration-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3b9eb02a..29d6184a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -113,11 +113,9 @@ jobs: - name: "Check out repository" uses: actions/checkout@v4 - - name: Setup `hatch` - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - name: "Test psycopg2 name - default" run: | + pip install --upgrade --force-reinstall . PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) echo $PSYCOPG2_PIP_ENTRY @@ -132,6 +130,7 @@ jobs: - name: "Test psycopg2 name - invalid override" run: | + pip install --upgrade --force-reinstall . PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) echo $PSYCOPG2_PIP_ENTRY @@ -147,6 +146,7 @@ jobs: - name: "Test psycopg2 name - override" run: | + pip install --upgrade --force-reinstall . PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) echo $PSYCOPG2_PIP_ENTRY From eb341545887edff62c46bf99d6f1f70ee966ca78 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 12:41:22 -0400 Subject: [PATCH 16/41] remove hatch build hook that is not working as expected --- hatch_hooks.py | 30 ------------------------------ pyproject.toml | 3 --- 2 files changed, 33 deletions(-) delete mode 100644 hatch_hooks.py diff --git a/hatch_hooks.py b/hatch_hooks.py deleted file mode 100644 index 9a4cabef..00000000 --- a/hatch_hooks.py +++ /dev/null @@ -1,30 +0,0 @@ -import os - -from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.plugin import hookimpl - - -class Psycopg2NoBinary(BuildHookInterface): - """ - Custom build hook to install psycopg2 instead of psycopg2-binary based on the presence of `DBT_PSYCOPG2_NAME`. - This is necessary as psycopg2-binary is better for local development, but psycopg2 is better for production. - """ - - PLUGIN_NAME = "psycopg2" - - def initialize(self, version, build_data) -> None: - print(build_data) - if os.getenv("DBT_PSYCOPG2_NAME", "") == "psycopg2": - psycopg2_binary_pinned = [ - package - for package in build_data["dependencies"] - if package.startswith("psycopg2-binary") - ].pop() - build_data["dependencies"].pop(psycopg2_binary_pinned) - psycopg2_pinned = psycopg2_binary_pinned.replace("-binary", "") - build_data["dependencies"].append(psycopg2_pinned) - - -@hookimpl -def hatch_register_build_hook(): - return Psycopg2NoBinary diff --git a/pyproject.toml b/pyproject.toml index 3110e201..92fbf82a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,9 +43,6 @@ Changelog = "https://github.com/dbt-labs/dbt-postgres/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" -[tool.hatch.build.hooks.custom] -path = "./hatch_hooks.py" - [tool.hatch.build.targets.sdist] include = ["dbt"] From ac937f4c29d0a7ae3d5b7b08d91f85aea293cf4b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 12:42:00 -0400 Subject: [PATCH 17/41] simplify psycopg2 check and override expected psycopg2 version since it is not implemented --- .github/workflows/integration-tests.yml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 29d6184a..2b2fb884 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -116,11 +116,7 @@ jobs: - name: "Test psycopg2 name - default" run: | pip install --upgrade --force-reinstall . - PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) - - echo $PSYCOPG2_PIP_ENTRY - PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" - echo $PSYCOPG2_NAME + PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 @@ -131,11 +127,7 @@ jobs: - name: "Test psycopg2 name - invalid override" run: | pip install --upgrade --force-reinstall . - PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) - - echo $PSYCOPG2_PIP_ENTRY - PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" - echo $PSYCOPG2_NAME + PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 @@ -147,15 +139,11 @@ jobs: - name: "Test psycopg2 name - override" run: | pip install --upgrade --force-reinstall . - PSYCOPG2_PIP_ENTRY=$(hatch run pip list | grep "psycopg2 " || hatch run pip list | grep psycopg2-binary) - - echo $PSYCOPG2_PIP_ENTRY - PSYCOPG2_NAME="${PSYCOPG2_PIP_ENTRY%% *}" - echo $PSYCOPG2_NAME + PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi env: DBT_PSYCOPG2_NAME: psycopg2 - PSYCOPG2_EXPECTED_NAME: psycopg2 + PSYCOPG2_EXPECTED_NAME: psycopg2-binary # this is not yet implemented From 1ffb3cb58a4538f48046707f0e2dc1252ce62a08 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 12:45:05 -0400 Subject: [PATCH 18/41] use hatch envs for psycopg2 test --- .github/workflows/integration-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 2b2fb884..3093a7c4 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -115,7 +115,7 @@ jobs: - name: "Test psycopg2 name - default" run: | - pip install --upgrade --force-reinstall . + hatch env prune PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then @@ -126,7 +126,7 @@ jobs: - name: "Test psycopg2 name - invalid override" run: | - pip install --upgrade --force-reinstall . + hatch env prune PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then @@ -138,7 +138,7 @@ jobs: - name: "Test psycopg2 name - override" run: | - pip install --upgrade --force-reinstall . + hatch env prune PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then From b0d55530e7afaef5d5fdaed5c843009048192ceb Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 13:05:44 -0400 Subject: [PATCH 19/41] add manual workaround and associated test --- .github/workflows/integration-tests.yml | 34 ++++++++++++++++++++----- README.md | 12 +++++++++ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3093a7c4..40211b66 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -106,17 +106,21 @@ jobs: strategy: fail-fast: false matrix: - platform: - - ubuntu-24.04 - - macos-12 + platform: [ubuntu-24.04, macos-12] + python-version: ["3.8", "3.11"] steps: - name: "Check out repository" uses: actions/checkout@v4 + - name: Setup `hatch` + uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + with: + python-version: ${{ matrix.python-version }} + - name: "Test psycopg2 name - default" run: | hatch env prune - PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$(hatch run pip show psycopg2 || hatch run pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 @@ -127,7 +131,7 @@ jobs: - name: "Test psycopg2 name - invalid override" run: | hatch env prune - PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$(hatch run pip show psycopg2 || hatch run pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 @@ -139,7 +143,7 @@ jobs: - name: "Test psycopg2 name - override" run: | hatch env prune - PSYCOPG2_NAME=$(hatch run pip show psycopg2 || pip show psycopg2-binary | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$(hatch run pip show psycopg2 || hatch run pip show psycopg2-binary | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 @@ -147,3 +151,21 @@ jobs: env: DBT_PSYCOPG2_NAME: psycopg2 PSYCOPG2_EXPECTED_NAME: psycopg2-binary # this is not yet implemented + + - name: "Test psycopg2 name - manual override" + run: | + hatch env prune + + # this is the work around + PSYCOPG2_VERSION = $(hatch run pip show psycopg2-binary | grep Version | cut -d " " -f 2) + hatch run pip uninstall -y psycopg2-binary + hatch run pip install psycopg2==$PSYCOPG2_VERSION + # this is the end of the work around + + PSYCOPG2_NAME=$(hatch run pip show psycopg2 || hatch run pip show psycopg2-binary | grep Name | cut -d " " -f 2) + + if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then + exit 1 + fi + env: + PSYCOPG2_EXPECTED_NAME: psycopg2 diff --git a/README.md b/README.md index d5b8900a..3462fe86 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,18 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get - [Install dbt](https://docs.getdbt.com/docs/installation) - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) +### `psycopg2-binary` vs. `psycopg2` + +By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps: + +```bash +PSYCOPG2_VERSION = $(pip show psycopg2-binary | grep Version | cut -d " " -f 2) +pip uninstall -y psycopg2-binary +pip install psycopg2==$PSYCOPG2_VERSION +``` + +This ensures the version of `psycopg2` will match that of `psycopg2-binary`. + ## Join the dbt Community - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) From a02bd81f8a5856d80a7092b67e5856c8e2742ea5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 13:09:07 -0400 Subject: [PATCH 20/41] roll ubuntu image back to 22.04 to support python 3.8 --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 40211b66..abcb5541 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -45,7 +45,7 @@ defaults: jobs: integration: name: Integration Tests - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: fail-fast: false @@ -106,7 +106,7 @@ jobs: strategy: fail-fast: false matrix: - platform: [ubuntu-24.04, macos-12] + platform: [ubuntu-22.04, macos-12] python-version: ["3.8", "3.11"] steps: - name: "Check out repository" From 08ed81419f7ff0dcfcd20fc34fcdc65ff19ae776 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 13:46:27 -0400 Subject: [PATCH 21/41] fix syntax error in test --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index abcb5541..c6217d4b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -157,7 +157,7 @@ jobs: hatch env prune # this is the work around - PSYCOPG2_VERSION = $(hatch run pip show psycopg2-binary | grep Version | cut -d " " -f 2) + PSYCOPG2_VERSION=$(hatch run pip show psycopg2-binary | grep Version | cut -d " " -f 2) hatch run pip uninstall -y psycopg2-binary hatch run pip install psycopg2==$PSYCOPG2_VERSION # this is the end of the work around From 4808e0b128e3a65d66f3b3d66c598c88b80f022d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 13:47:36 -0400 Subject: [PATCH 22/41] fix the typo in the workaround in the readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3462fe86..09870d7d 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps: ```bash -PSYCOPG2_VERSION = $(pip show psycopg2-binary | grep Version | cut -d " " -f 2) +PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) pip uninstall -y psycopg2-binary pip install psycopg2==$PSYCOPG2_VERSION ``` From 991f0662c8d0759878fd900592be3fd3e63ecadc Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 7 Jun 2024 14:08:45 -0400 Subject: [PATCH 23/41] fix syntax to resolve || before | --- .github/workflows/integration-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c6217d4b..845edca0 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -120,7 +120,7 @@ jobs: - name: "Test psycopg2 name - default" run: | hatch env prune - PSYCOPG2_NAME=$(hatch run pip show psycopg2 || hatch run pip show psycopg2-binary | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 @@ -131,7 +131,7 @@ jobs: - name: "Test psycopg2 name - invalid override" run: | hatch env prune - PSYCOPG2_NAME=$(hatch run pip show psycopg2 || hatch run pip show psycopg2-binary | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 @@ -143,7 +143,7 @@ jobs: - name: "Test psycopg2 name - override" run: | hatch env prune - PSYCOPG2_NAME=$(hatch run pip show psycopg2 || hatch run pip show psycopg2-binary | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 @@ -162,7 +162,7 @@ jobs: hatch run pip install psycopg2==$PSYCOPG2_VERSION # this is the end of the work around - PSYCOPG2_NAME=$(hatch run pip show psycopg2 || hatch run pip show psycopg2-binary | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 From cc891c0dd98b1ae6af25f2c4f9afde5e627eb836 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jun 2024 17:36:00 -0400 Subject: [PATCH 24/41] add post install command to swap out psycopg2 --- .github/workflows/integration-tests.yml | 2 +- pyproject.toml | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 845edca0..60918c5b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -150,7 +150,7 @@ jobs: fi env: DBT_PSYCOPG2_NAME: psycopg2 - PSYCOPG2_EXPECTED_NAME: psycopg2-binary # this is not yet implemented + PSYCOPG2_EXPECTED_NAME: psycopg2 - name: "Test psycopg2 name - manual override" run: | diff --git a/pyproject.toml b/pyproject.toml index 92fbf82a..c9e40179 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,10 +44,10 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.sdist] -include = ["dbt"] +include = ["dbt", "scripts"] [tool.hatch.build.targets.wheel] -packages = ["dbt"] +packages = ["dbt", "scripts"] [tool.hatch.version] path = "dbt/adapters/postgres/__version__.py" @@ -59,6 +59,13 @@ dependencies = [ 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', ] +post-install-commands = [""" + if [[ $DBT_PSYCOPG2_NAME == 'psycopg2' ]]; then + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION + fi""", +] [tool.hatch.envs.default.scripts] dev = "pre-commit install" code-quality = "pre-commit run --all-files" From 2fc1c020fed29ceb6574b477b1bd0bdc49bca8d2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jun 2024 17:58:18 -0400 Subject: [PATCH 25/41] use full install instead of editable install --- .github/workflows/integration-tests.yml | 41 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 60918c5b..96a2a06b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -119,53 +119,72 @@ jobs: - name: "Test psycopg2 name - default" run: | - hatch env prune - PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) + python -m venv venv + ./venv/bin/activate + python -m pip install . + PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi + + deactivate + rmdir -r ./venv env: PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 name - invalid override" run: | - hatch env prune - PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) + python -m venv venv + ./venv/bin/activate + python -m pip install . + PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi + + deactivate + rmdir -r ./venv env: DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 name - override" run: | - hatch env prune - PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) + python -m venv venv + ./venv/bin/activate + python -m pip install . + PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi + + deactivate + rmdir -r ./venv env: DBT_PSYCOPG2_NAME: psycopg2 PSYCOPG2_EXPECTED_NAME: psycopg2 - name: "Test psycopg2 name - manual override" run: | - hatch env prune + python -m venv venv + ./venv/bin/activate + python -m pip install . # this is the work around - PSYCOPG2_VERSION=$(hatch run pip show psycopg2-binary | grep Version | cut -d " " -f 2) - hatch run pip uninstall -y psycopg2-binary - hatch run pip install psycopg2==$PSYCOPG2_VERSION + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION # this is the end of the work around PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) - if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi + + deactivate + rmdir -r ./venv env: PSYCOPG2_EXPECTED_NAME: psycopg2 From 6f51bcd7b4f75ef2ac9f1b827c0fb25072bfdca5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 12 Jun 2024 18:07:11 -0400 Subject: [PATCH 26/41] fix venv activate syntax --- .github/workflows/integration-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 96a2a06b..b43d7abd 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -120,7 +120,7 @@ jobs: - name: "Test psycopg2 name - default" run: | python -m venv venv - ./venv/bin/activate + source venv/bin/activate python -m pip install . PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) @@ -136,7 +136,7 @@ jobs: - name: "Test psycopg2 name - invalid override" run: | python -m venv venv - ./venv/bin/activate + source venv/bin/activate python -m pip install . PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) @@ -153,7 +153,7 @@ jobs: - name: "Test psycopg2 name - override" run: | python -m venv venv - ./venv/bin/activate + source venv/bin/activate python -m pip install . PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) @@ -170,7 +170,7 @@ jobs: - name: "Test psycopg2 name - manual override" run: | python -m venv venv - ./venv/bin/activate + source venv/bin/activate python -m pip install . # this is the work around From 7d00964585b50f19682d60a1f78b7b8a3fe4827d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 13:16:52 -0400 Subject: [PATCH 27/41] fix rm syntax --- .github/workflows/integration-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b43d7abd..e1539a98 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -129,7 +129,7 @@ jobs: fi deactivate - rmdir -r ./venv + rm -r ./venv env: PSYCOPG2_EXPECTED_NAME: psycopg2-binary @@ -145,7 +145,7 @@ jobs: fi deactivate - rmdir -r ./venv + rm -r ./venv env: DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers PSYCOPG2_EXPECTED_NAME: psycopg2-binary @@ -162,7 +162,7 @@ jobs: fi deactivate - rmdir -r ./venv + rm -r ./venv env: DBT_PSYCOPG2_NAME: psycopg2 PSYCOPG2_EXPECTED_NAME: psycopg2 @@ -185,6 +185,6 @@ jobs: fi deactivate - rmdir -r ./venv + rm -r ./venv env: PSYCOPG2_EXPECTED_NAME: psycopg2 From 2c7d516e955e0a23bbc07a282bc7c0f05332cfac Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 13:26:03 -0400 Subject: [PATCH 28/41] remove references to hatch in psycopg2 test --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e1539a98..c4b6d4c9 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -156,7 +156,7 @@ jobs: source venv/bin/activate python -m pip install . - PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi @@ -179,7 +179,7 @@ jobs: pip install psycopg2==$PSYCOPG2_VERSION # this is the end of the work around - PSYCOPG2_NAME=$((hatch run pip show psycopg2 || hatch run pip show psycopg2-binary) | grep Name | cut -d " " -f 2) + PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then exit 1 fi From 2c1ee82715db70cf000ed57ea2cdccd36be673d4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 15:15:59 -0400 Subject: [PATCH 29/41] remove inapplicable tests --- tests/functional/test_selection/conftest.py | 96 --- .../test_selection_expansion.py | 567 ------------------ 2 files changed, 663 deletions(-) delete mode 100644 tests/functional/test_selection/conftest.py delete mode 100644 tests/functional/test_selection/test_selection_expansion.py diff --git a/tests/functional/test_selection/conftest.py b/tests/functional/test_selection/conftest.py deleted file mode 100644 index 2faa9e34..00000000 --- a/tests/functional/test_selection/conftest.py +++ /dev/null @@ -1,96 +0,0 @@ -from dbt.tests.fixtures.project import write_project_files -import pytest - - -tests__cf_a_b_sql = """ -select * from {{ ref('model_a') }} -cross join {{ ref('model_b') }} -where false -""" - -tests__cf_a_src_sql = """ -select * from {{ ref('model_a') }} -cross join {{ source('my_src', 'my_tbl') }} -where false -""" - -tests__just_a_sql = """ -{{ config(tags = ['data_test_tag']) }} - -select * from {{ ref('model_a') }} -where false -""" - -models__schema_yml = """ -version: 2 - -sources: - - name: my_src - schema: "{{ target.schema }}" - tables: - - name: my_tbl - identifier: model_b - columns: - - name: fun - data_tests: - - unique - -models: - - name: model_a - columns: - - name: fun - tags: [column_level_tag] - data_tests: - - unique - - relationships: - to: ref('model_b') - field: fun - tags: [test_level_tag] - - relationships: - to: source('my_src', 'my_tbl') - field: fun -""" - -models__model_b_sql = """ -{{ config( - tags = ['a_or_b'] -) }} - -select 1 as fun -""" - -models__model_a_sql = """ -{{ config( - tags = ['a_or_b'] -) }} - -select * FROM {{ref('model_b')}} -""" - - -@pytest.fixture(scope="class") -def tests(): - return { - "cf_a_b.sql": tests__cf_a_b_sql, - "cf_a_src.sql": tests__cf_a_src_sql, - "just_a.sql": tests__just_a_sql, - } - - -@pytest.fixture(scope="class") -def models(): - return { - "schema.yml": models__schema_yml, - "model_b.sql": models__model_b_sql, - "model_a.sql": models__model_a_sql, - } - - -@pytest.fixture(scope="class") -def project_files( - project_root, - tests, - models, -): - write_project_files(project_root, "tests", tests) - write_project_files(project_root, "models", models) diff --git a/tests/functional/test_selection/test_selection_expansion.py b/tests/functional/test_selection/test_selection_expansion.py deleted file mode 100644 index d17f27d7..00000000 --- a/tests/functional/test_selection/test_selection_expansion.py +++ /dev/null @@ -1,567 +0,0 @@ -from dbt.tests.util import run_dbt -import pytest - - -class TestSelectionExpansion: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"config-version": 2, "test-paths": ["tests"]} - - def list_tests_and_assert( - self, - include, - exclude, - expected_tests, - indirect_selection="eager", - selector_name=None, - ): - list_args = ["ls", "--resource-type", "test"] - if include: - list_args.extend(("--select", include)) - if exclude: - list_args.extend(("--exclude", exclude)) - if indirect_selection: - list_args.extend(("--indirect-selection", indirect_selection)) - if selector_name: - list_args.extend(("--selector", selector_name)) - - listed = run_dbt(list_args) - assert len(listed) == len(expected_tests) - - test_names = [name.split(".")[-1] for name in listed] - assert sorted(test_names) == sorted(expected_tests) - - def run_tests_and_assert( - self, - include, - exclude, - expected_tests, - indirect_selection="eager", - selector_name=None, - ): - results = run_dbt(["run"]) - assert len(results) == 2 - - test_args = ["test"] - if include: - test_args.extend(("--models", include)) - if exclude: - test_args.extend(("--exclude", exclude)) - if indirect_selection: - test_args.extend(("--indirect-selection", indirect_selection)) - if selector_name: - test_args.extend(("--selector", selector_name)) - - results = run_dbt(test_args) - tests_run = [r.node.name for r in results] - assert len(tests_run) == len(expected_tests) - - assert sorted(tests_run) == sorted(expected_tests) - - def test_all_tests_no_specifiers( - self, - project, - ): - select = None - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_alone( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_model_b( - self, - project, - ): - select = "model_a model_b" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "unique_model_a_fun", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_sources( - self, - project, - ): - select = "model_a source:*" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "unique_model_a_fun", - "source_unique_my_src_my_tbl_fun", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_model_b( - self, - project, - ): - select = None - exclude = "model_b" - expected = [ - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_exclude_specific_test( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_exclude_specific_test_cautious( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = ["just_a"] - indirect_selection = "cautious" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_exclude_specific_test_buildable( - self, - project, - ): - select = "model_a" - exclude = "unique_model_a_fun" - expected = [ - "just_a", - "cf_a_b", - "cf_a_src", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - indirect_selection = "buildable" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_only_generic( - self, - project, - ): - select = "test_type:generic" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "source_unique_my_src_my_tbl_fun", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_unset( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_eager( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular_cautious( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["just_a"] - indirect_selection = "cautious" - - self.list_tests_and_assert( - select, exclude, expected, indirect_selection=indirect_selection - ) - self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection) - - def test_only_singular( - self, - project, - ): - select = "test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_only_singular( - self, - project, - ): - select = "model_a,test_type:singular" - exclude = None - expected = ["cf_a_b", "cf_a_src", "just_a"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_test_name_intersection( - self, - project, - ): - select = "model_a,test_name:unique" - exclude = None - expected = ["unique_model_a_fun"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_tag_test_name_intersection( - self, - project, - ): - select = "tag:a_or_b,test_name:relationships" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_select_column_level_tag( - self, - project, - ): - select = "tag:column_level_tag" - exclude = None - expected = [ - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_column_level_tag( - self, - project, - ): - select = None - exclude = "tag:column_level_tag" - expected = ["cf_a_b", "cf_a_src", "just_a", "source_unique_my_src_my_tbl_fun"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_test_level_tag( - self, - project, - ): - select = "tag:test_level_tag" - exclude = None - expected = ["relationships_model_a_fun__fun__ref_model_b_"] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_exclude_data_test_tag( - self, - project, - ): - select = "model_a" - exclude = "tag:data_test_tag" - expected = [ - "cf_a_b", - "cf_a_src", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_indirect_selection( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert(select, exclude, expected) - self.run_tests_and_assert(select, exclude, expected) - - def test_model_a_indirect_selection_eager( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - indirect_selection = "eager" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_cautious( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "just_a", - "unique_model_a_fun", - ] - indirect_selection = "cautious" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_buildable( - self, - project, - ): - select = "model_a" - exclude = None - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - indirect_selection = "buildable" - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection) - - def test_model_a_indirect_selection_exclude_unique_tests( - self, - project, - ): - select = "model_a" - exclude = "test_name:unique" - indirect_selection = "eager" - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - ] - - self.list_tests_and_assert(select, exclude, expected, indirect_selection) - self.run_tests_and_assert(select, exclude, expected, indirect_selection=indirect_selection) - - def test_model_a_indirect_selection_empty(self, project): - results = run_dbt(["ls", "--indirect-selection", "empty", "--select", "model_a"]) - assert len(results) == 1 - - -class TestExpansionWithSelectors(TestSelectionExpansion): - @pytest.fixture(scope="class") - def selectors(self): - return """ - selectors: - - name: model_a_unset_indirect_selection - definition: - method: fqn - value: model_a - - name: model_a_cautious_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "cautious" - - name: model_a_eager_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "eager" - - name: model_a_buildable_indirect_selection - definition: - method: fqn - value: model_a - indirect_selection: "buildable" - """ - - def test_selector_model_a_unset_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_unset_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_unset_indirect_selection", - ) - - def test_selector_model_a_cautious_indirect_selection( - self, - project, - ): - expected = ["just_a", "unique_model_a_fun"] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_cautious_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_cautious_indirect_selection", - ) - - def test_selector_model_a_eager_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_eager_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_eager_indirect_selection", - ) - - def test_selector_model_a_buildable_indirect_selection( - self, - project, - ): - expected = [ - "cf_a_b", - "cf_a_src", - "just_a", - "relationships_model_a_fun__fun__ref_model_b_", - "relationships_model_a_fun__fun__source_my_src_my_tbl_", - "unique_model_a_fun", - ] - - self.list_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_buildable_indirect_selection", - ) - self.run_tests_and_assert( - include=None, - exclude=None, - expected_tests=expected, - selector_name="model_a_buildable_indirect_selection", - ) From 796aa414939c007cccc4d533bab4db9f3d979116 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 17:40:08 -0400 Subject: [PATCH 30/41] swap out post-install-command for a build hook --- hatch_build.py | 30 ++++++++++++++++++++++++++++++ pyproject.toml | 10 +++------- 2 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 hatch_build.py diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 00000000..a45b6201 --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,30 @@ +import os + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.plugin import hookimpl + + +class Psycopg2BuildHook(BuildHookInterface): + """ + Replace `psycopg2-binary` with the same version of `psycopg2` + when configured via `DBT_PSYCOPG2_NAME=psycopg2` + """ + + def initialize(self, version, build_data): + if os.getenv("DBT_PSYCOPG2_NAME") == "psycopg2": + psycopg2_binary = [ + dep + for dep in self.metadata._core.config.get("dependencies") + if dep.startswith("psycopg2-binary") + ].pop() + if psycopg2_binary: + psycopg2_no_binary = psycopg2_binary.replace("-binary", "") + self.metadata._core.config["dependencies"].remove(psycopg2_binary) + self.metadata._core.config["dependencies"].append(psycopg2_no_binary) + else: + raise ImportError("`psycopg2-binary not found") + + +@hookimpl +def hatch_register_build_hook(): + return Psycopg2BuildHook diff --git a/pyproject.toml b/pyproject.toml index c9e40179..54a8fa17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,9 @@ Changelog = "https://github.com/dbt-labs/dbt-postgres/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" +[tool.hatch.build.hooks.custom] +path = "hatch_build.py" + [tool.hatch.build.targets.sdist] include = ["dbt", "scripts"] @@ -59,13 +62,6 @@ dependencies = [ 'pre-commit==3.7.0;python_version>="3.9"', 'pre-commit==3.5.0;python_version=="3.8"', ] -post-install-commands = [""" - if [[ $DBT_PSYCOPG2_NAME == 'psycopg2' ]]; then - PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) - pip uninstall -y psycopg2-binary - pip install psycopg2==$PSYCOPG2_VERSION - fi""", -] [tool.hatch.envs.default.scripts] dev = "pre-commit install" code-quality = "pre-commit run --all-files" From 721e515629bcb91603903ce985241993a49c61a6 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 18:01:48 -0400 Subject: [PATCH 31/41] add build post hook --- hatch_build.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index a45b6201..d65fbb5f 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -1,4 +1,7 @@ +from importlib.metadata import version as get_version import os +import subprocess +from typing import Any, Dict from hatchling.builders.hooks.plugin.interface import BuildHookInterface from hatchling.plugin import hookimpl @@ -10,19 +13,11 @@ class Psycopg2BuildHook(BuildHookInterface): when configured via `DBT_PSYCOPG2_NAME=psycopg2` """ - def initialize(self, version, build_data): + def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None: if os.getenv("DBT_PSYCOPG2_NAME") == "psycopg2": - psycopg2_binary = [ - dep - for dep in self.metadata._core.config.get("dependencies") - if dep.startswith("psycopg2-binary") - ].pop() - if psycopg2_binary: - psycopg2_no_binary = psycopg2_binary.replace("-binary", "") - self.metadata._core.config["dependencies"].remove(psycopg2_binary) - self.metadata._core.config["dependencies"].append(psycopg2_no_binary) - else: - raise ImportError("`psycopg2-binary not found") + psycopg2_version = get_version("psycopg2-binary") + subprocess.run("pip", "uninstall", "-y", "psycopg2-binary") + subprocess.run("pip", "install", f"psycopg2=={psycopg2_version}") @hookimpl From 7e830ba8782f240dd0fed78f9519ee19f984082e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 18:19:17 -0400 Subject: [PATCH 32/41] remove all hatch hooks --- .github/workflows/integration-tests.yml | 2 +- hatch_build.py | 25 ------------------------- pyproject.toml | 3 --- 3 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 hatch_build.py diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c4b6d4c9..9c135385 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -165,7 +165,7 @@ jobs: rm -r ./venv env: DBT_PSYCOPG2_NAME: psycopg2 - PSYCOPG2_EXPECTED_NAME: psycopg2 + PSYCOPG2_EXPECTED_NAME: psycopg2-binary # we have not implemented the hook yet, so this doesn't work - name: "Test psycopg2 name - manual override" run: | diff --git a/hatch_build.py b/hatch_build.py deleted file mode 100644 index d65fbb5f..00000000 --- a/hatch_build.py +++ /dev/null @@ -1,25 +0,0 @@ -from importlib.metadata import version as get_version -import os -import subprocess -from typing import Any, Dict - -from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.plugin import hookimpl - - -class Psycopg2BuildHook(BuildHookInterface): - """ - Replace `psycopg2-binary` with the same version of `psycopg2` - when configured via `DBT_PSYCOPG2_NAME=psycopg2` - """ - - def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None: - if os.getenv("DBT_PSYCOPG2_NAME") == "psycopg2": - psycopg2_version = get_version("psycopg2-binary") - subprocess.run("pip", "uninstall", "-y", "psycopg2-binary") - subprocess.run("pip", "install", f"psycopg2=={psycopg2_version}") - - -@hookimpl -def hatch_register_build_hook(): - return Psycopg2BuildHook diff --git a/pyproject.toml b/pyproject.toml index 54a8fa17..40411863 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,9 +43,6 @@ Changelog = "https://github.com/dbt-labs/dbt-postgres/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" -[tool.hatch.build.hooks.custom] -path = "hatch_build.py" - [tool.hatch.build.targets.sdist] include = ["dbt", "scripts"] From f17d448a0772e19a9cdf44041cbbc9262be93f8f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 18:24:48 -0400 Subject: [PATCH 33/41] revert dev changes in pyproject.toml --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 40411863..92fbf82a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,10 +44,10 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.sdist] -include = ["dbt", "scripts"] +include = ["dbt"] [tool.hatch.build.targets.wheel] -packages = ["dbt", "scripts"] +packages = ["dbt"] [tool.hatch.version] path = "dbt/adapters/postgres/__version__.py" From 52cd912dfb14109f33f63f546a08c672b63acb74 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 13 Jun 2024 18:29:25 -0400 Subject: [PATCH 34/41] remove hatch as a test req for psycopg2 --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 9c135385..85a21b92 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -112,8 +112,8 @@ jobs: - name: "Check out repository" uses: actions/checkout@v4 - - name: Setup `hatch` - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + - name: "Set up Python ${{ matrix.python-version }}" + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} From af57f2e5b2125ab868c5f8634743797f539f4766 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 14 Jun 2024 12:29:06 -0400 Subject: [PATCH 35/41] move the script into a separate file to be reused --- .github/scripts/psycopg2-check.sh | 18 ++++++++ .github/workflows/integration-tests.yml | 60 +++---------------------- 2 files changed, 24 insertions(+), 54 deletions(-) create mode 100644 .github/scripts/psycopg2-check.sh diff --git a/.github/scripts/psycopg2-check.sh b/.github/scripts/psycopg2-check.sh new file mode 100644 index 00000000..e00e1c58 --- /dev/null +++ b/.github/scripts/psycopg2-check.sh @@ -0,0 +1,18 @@ +python -m venv venv +source venv/bin/activate +python -m pip install . + +if [[ "$PSYCOPG2_WORKAROUND" == true ]]; then + PSYCOPG2_VERSION=$(pip show --format=json | jq -r '.[] | select(.Name == "psycopg2-binary") | .Version') + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION +fi + +PSYCOPG2_NAME=$(pip show --format=json | jq -r '.[] | select(.Name == "psycopg2" or .Name == "psycopg2-binary") | .name') +if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then + echo -e "Expected: `$PSYCOPG2_EXPECTED_NAME` but found: `$PSYCOPG2_NAME`" + exit 1 +fi + +deactivate +rm -r ./venv diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 85a21b92..dcc49698 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -118,73 +118,25 @@ jobs: python-version: ${{ matrix.python-version }} - name: "Test psycopg2 name - default" - run: | - python -m venv venv - source venv/bin/activate - python -m pip install . - - PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) - if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then - exit 1 - fi - - deactivate - rm -r ./venv + run: ./.github/scripts/psycopg2-check.sh env: PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 name - invalid override" - run: | - python -m venv venv - source venv/bin/activate - python -m pip install . - - PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) - if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then - exit 1 - fi - - deactivate - rm -r ./venv + run: ./.github/scripts/psycopg2-check.sh env: DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 name - override" - run: | - python -m venv venv - source venv/bin/activate - python -m pip install . - - PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) - if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then - exit 1 - fi - - deactivate - rm -r ./venv + run: ./.github/scripts/psycopg2-check.sh env: DBT_PSYCOPG2_NAME: psycopg2 PSYCOPG2_EXPECTED_NAME: psycopg2-binary # we have not implemented the hook yet, so this doesn't work - name: "Test psycopg2 name - manual override" - run: | - python -m venv venv - source venv/bin/activate - python -m pip install . - - # this is the work around - PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) - pip uninstall -y psycopg2-binary - pip install psycopg2==$PSYCOPG2_VERSION - # this is the end of the work around - - PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) - if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then - exit 1 - fi - - deactivate - rm -r ./venv + # verify that the workaround documented in the `README.md` continues to work + run: ./.github/scripts/psycopg2-check.sh env: + PSYCOPG2_WORKAROUND: true PSYCOPG2_EXPECTED_NAME: psycopg2 From 43b5342eef43c4ac8f6933d480834e7890213817 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 14 Jun 2024 12:29:25 -0400 Subject: [PATCH 36/41] update workaround to use jq over string parsing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 09870d7d..b317e0f8 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps: ```bash -PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) +PSYCOPG2_VERSION=$(pip show --format=json | jq -r '.[] | select(.Name == "psycopg2-binary") | .Version') pip uninstall -y psycopg2-binary pip install psycopg2==$PSYCOPG2_VERSION ``` From 3daa2d040e71a53ef186aa34e5c5e2809b214a38 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 14 Jun 2024 12:34:56 -0400 Subject: [PATCH 37/41] mark script as executable --- .github/scripts/psycopg2-check.sh | 0 .github/workflows/integration-tests.yml | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) mode change 100644 => 100755 .github/scripts/psycopg2-check.sh diff --git a/.github/scripts/psycopg2-check.sh b/.github/scripts/psycopg2-check.sh old mode 100644 new mode 100755 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index dcc49698..d7367190 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -118,25 +118,25 @@ jobs: python-version: ${{ matrix.python-version }} - name: "Test psycopg2 name - default" - run: ./.github/scripts/psycopg2-check.sh + run: .github/scripts/psycopg2-check.sh env: PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 name - invalid override" - run: ./.github/scripts/psycopg2-check.sh + run: .github/scripts/psycopg2-check.sh env: DBT_PSYCOPG2_NAME: rubber-baby-buggy-bumpers PSYCOPG2_EXPECTED_NAME: psycopg2-binary - name: "Test psycopg2 name - override" - run: ./.github/scripts/psycopg2-check.sh + run: .github/scripts/psycopg2-check.sh env: DBT_PSYCOPG2_NAME: psycopg2 PSYCOPG2_EXPECTED_NAME: psycopg2-binary # we have not implemented the hook yet, so this doesn't work - name: "Test psycopg2 name - manual override" # verify that the workaround documented in the `README.md` continues to work - run: ./.github/scripts/psycopg2-check.sh + run: .github/scripts/psycopg2-check.sh env: PSYCOPG2_WORKAROUND: true PSYCOPG2_EXPECTED_NAME: psycopg2 From 41e3126d2788b34808d2e8c0e87f810cc3bd806f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 14 Jun 2024 12:38:00 -0400 Subject: [PATCH 38/41] fix syntax in error message --- .github/scripts/psycopg2-check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/psycopg2-check.sh b/.github/scripts/psycopg2-check.sh index e00e1c58..8f43d485 100755 --- a/.github/scripts/psycopg2-check.sh +++ b/.github/scripts/psycopg2-check.sh @@ -10,7 +10,7 @@ fi PSYCOPG2_NAME=$(pip show --format=json | jq -r '.[] | select(.Name == "psycopg2" or .Name == "psycopg2-binary") | .name') if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then - echo -e "Expected: `$PSYCOPG2_EXPECTED_NAME` but found: `$PSYCOPG2_NAME`" + echo -e 'Expected: "$PSYCOPG2_EXPECTED_NAME" but found: "$PSYCOPG2_NAME"' exit 1 fi From d2fd8f932b029b0d029a2fa043a318219089d78d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 14 Jun 2024 13:09:24 -0400 Subject: [PATCH 39/41] revert to string parsing to avoid jq dep --- .github/scripts/psycopg2-check.sh | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/psycopg2-check.sh b/.github/scripts/psycopg2-check.sh index 8f43d485..ca67b7dc 100755 --- a/.github/scripts/psycopg2-check.sh +++ b/.github/scripts/psycopg2-check.sh @@ -3,12 +3,12 @@ source venv/bin/activate python -m pip install . if [[ "$PSYCOPG2_WORKAROUND" == true ]]; then - PSYCOPG2_VERSION=$(pip show --format=json | jq -r '.[] | select(.Name == "psycopg2-binary") | .Version') + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) pip uninstall -y psycopg2-binary pip install psycopg2==$PSYCOPG2_VERSION fi -PSYCOPG2_NAME=$(pip show --format=json | jq -r '.[] | select(.Name == "psycopg2" or .Name == "psycopg2-binary") | .name') +PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) if [[ "$PSYCOPG2_NAME" != "$PSYCOPG2_EXPECTED_NAME" ]]; then echo -e 'Expected: "$PSYCOPG2_EXPECTED_NAME" but found: "$PSYCOPG2_NAME"' exit 1 diff --git a/README.md b/README.md index b317e0f8..09870d7d 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps: ```bash -PSYCOPG2_VERSION=$(pip show --format=json | jq -r '.[] | select(.Name == "psycopg2-binary") | .Version') +PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) pip uninstall -y psycopg2-binary pip install psycopg2==$PSYCOPG2_VERSION ``` From d3b0cc5ea9c3f3e49b9b4f37e34050d9f3a9ac88 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 17 Jun 2024 13:44:17 -0400 Subject: [PATCH 40/41] update manual workaround to be more robust to future changes --- .github/scripts/psycopg2-check.sh | 8 +++++--- README.md | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/scripts/psycopg2-check.sh b/.github/scripts/psycopg2-check.sh index ca67b7dc..a3e99ada 100755 --- a/.github/scripts/psycopg2-check.sh +++ b/.github/scripts/psycopg2-check.sh @@ -3,9 +3,11 @@ source venv/bin/activate python -m pip install . if [[ "$PSYCOPG2_WORKAROUND" == true ]]; then - PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) - pip uninstall -y psycopg2-binary - pip install psycopg2==$PSYCOPG2_VERSION + if [[ pip show psycopg2-binary ]]; then + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION + fi fi PSYCOPG2_NAME=$((pip show psycopg2 || pip show psycopg2-binary) | grep Name | cut -d " " -f 2) diff --git a/README.md b/README.md index 09870d7d..2e66dc64 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,11 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps: ```bash -PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) -pip uninstall -y psycopg2-binary -pip install psycopg2==$PSYCOPG2_VERSION +if [[ pip show psycopg2-binary ]]; then + PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) + pip uninstall -y psycopg2-binary + pip install psycopg2==$PSYCOPG2_VERSION +fi ``` This ensures the version of `psycopg2` will match that of `psycopg2-binary`. From 36f94d76bb6105e6d62281c66bde74d32874a99b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 17 Jun 2024 13:46:45 -0400 Subject: [PATCH 41/41] fix syntax --- .github/scripts/psycopg2-check.sh | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/psycopg2-check.sh b/.github/scripts/psycopg2-check.sh index a3e99ada..faee902c 100755 --- a/.github/scripts/psycopg2-check.sh +++ b/.github/scripts/psycopg2-check.sh @@ -3,7 +3,7 @@ source venv/bin/activate python -m pip install . if [[ "$PSYCOPG2_WORKAROUND" == true ]]; then - if [[ pip show psycopg2-binary ]]; then + if [[ $(pip show psycopg2-binary) ]]; then PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) pip uninstall -y psycopg2-binary pip install psycopg2==$PSYCOPG2_VERSION diff --git a/README.md b/README.md index 2e66dc64..285f5144 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ more information on using dbt with Postgres, consult [the docs](https://docs.get By default, `dbt-postgres` installs `psycopg2-binary`. This is great for development, and even testing, as it does not require any OS dependencies; it's a pre-built wheel. However, building `psycopg2` from source will grant performance improvements that are desired in a production environment. In order to install `psycopg2`, use the following steps: ```bash -if [[ pip show psycopg2-binary ]]; then +if [[ $(pip show psycopg2-binary) ]]; then PSYCOPG2_VERSION=$(pip show psycopg2-binary | grep Version | cut -d " " -f 2) pip uninstall -y psycopg2-binary pip install psycopg2==$PSYCOPG2_VERSION