From 88aff7c1d3ae1fd09b1487179f2da3acadfb42f4 Mon Sep 17 00:00:00 2001 From: Alena Date: Wed, 5 Jun 2024 12:05:22 +0200 Subject: [PATCH] delete manual duckdb extension installation --- Makefile | 4 ---- .../postgres_to_postgres/install_duckdb_extensions.py | 7 ------- .../examples/postgres_to_postgres/postgres_to_postgres.py | 8 +++++--- 3 files changed, 5 insertions(+), 14 deletions(-) delete mode 100644 docs/examples/postgres_to_postgres/install_duckdb_extensions.py diff --git a/Makefile b/Makefile index 81fae8e004..92985ee82f 100644 --- a/Makefile +++ b/Makefile @@ -70,10 +70,6 @@ lint-and-test-examples: poetry run mypy --config-file mypy.ini docs/examples poetry run flake8 --max-line-length=200 docs/examples cd docs/tools && poetry run python prepare_examples_tests.py - # install duckdb extension for postgres_to_postgres example - cd docs/examples/postgres_to_postgres && wget http://extensions.duckdb.org/v0.10.3/linux_amd64_gcc4/postgres_scanner.duckdb_extension.gz && gunzip ./postgres_scanner.duckdb_extension.gz - cd docs/examples/postgres_to_postgres && poetry run python ./install_duckdb_extensions.py - # run tests cd docs/examples && poetry run pytest diff --git a/docs/examples/postgres_to_postgres/install_duckdb_extensions.py b/docs/examples/postgres_to_postgres/install_duckdb_extensions.py deleted file mode 100644 index 912725cc4d..0000000000 --- a/docs/examples/postgres_to_postgres/install_duckdb_extensions.py +++ /dev/null @@ -1,7 +0,0 @@ -import duckdb - -conn = duckdb.connect() -conn.sql('from duckdb_extensions()').show() # checking if extensions are loaded -conn.install_extension("./postgres_scanner.duckdb_extension") # this is where I copied it manually -conn.sql("LOAD postgres;") -conn.sql('from duckdb_extensions()').show() # checking if extensions are loaded diff --git a/docs/examples/postgres_to_postgres/postgres_to_postgres.py b/docs/examples/postgres_to_postgres/postgres_to_postgres.py index 4ba8754975..85b8aed045 100644 --- a/docs/examples/postgres_to_postgres/postgres_to_postgres.py +++ b/docs/examples/postgres_to_postgres/postgres_to_postgres.py @@ -5,6 +5,10 @@ keywords: [connector x, pyarrow, zero copy, duckdb, postgres, initial load] --- +:::info +Huge shout out to [Simon Späti](https://github.com/sspaeti) for this example! +::: + This examples shows you how to export and import data from Postgres to Postgres in a fast way with ConnectorX and DuckDB since the default export will generate `Insert_statement` during the normalization phase, which is super slow for large tables. @@ -218,9 +222,7 @@ def table_desc(table_name, pk, schema_name, order_date, columns="*"): print("##################################### START DUCKDB LOAD ########") # connect to local duckdb dump conn = duckdb.connect(f"{load_info.destination_displayable_credentials}".split(":///")[1]) - conn.install_extension( - "./postgres_scanner.duckdb_extension" - ) # duckdb_extension is downloaded/installed manually here, only needed if `LOAD/INSTALL postgres` throws an error + conn.sql("INSTALL postgres;") conn.sql("LOAD postgres;") # select generated timestamp schema timestamped_schema = conn.sql(