From 417357cb31833df62ce4a28cab232379adee34ee Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Wed, 27 Sep 2023 22:22:40 +0200 Subject: [PATCH] forces local duckdb version when installing dbt venv --- dlt/helpers/dbt/__init__.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/dlt/helpers/dbt/__init__.py b/dlt/helpers/dbt/__init__.py index bef585386c..a68630dc01 100644 --- a/dlt/helpers/dbt/__init__.py +++ b/dlt/helpers/dbt/__init__.py @@ -43,18 +43,24 @@ def _create_dbt_deps(destination_names: List[str], dbt_version: str = DEFAULT_DB else: dbt_version = "" - all_packages = destination_names + ["core"] + # add version only to the core package. the other packages versions must be resolved by pip + all_packages = ["core" + dbt_version] + destination_names for idx, package in enumerate(all_packages): - # TODO: if we have more cases like this, move to destination capabilities - package = DBT_DESTINATION_MAP.get(package, package) - package_w_ver = "dbt-" + package + dbt_version + package = "dbt-" + DBT_DESTINATION_MAP.get(package, package) # verify package - pkg_resources.Requirement.parse(package_w_ver) - all_packages[idx] = package_w_ver + pkg_resources.Requirement.parse(package) + all_packages[idx] = package dlt_requirement = get_installed_requirement_string() - - return all_packages + [dlt_requirement] + # get additional requirements + additional_deps: List[str] = [] + if "duckdb" in destination_names or "motherduck" in destination_names: + from importlib.metadata import version as pkg_version + # force locally installed duckdb + additional_deps = ["duckdb" + "==" + pkg_version("duckdb")] + + print(all_packages + [dlt_requirement] + additional_deps) + return all_packages + [dlt_requirement] + additional_deps def restore_venv(venv_dir: str, destination_names: List[str], dbt_version: str = DEFAULT_DBT_VERSION) -> Venv: