From f64e3038414af36eb59e8c604e90351863ebd4fa Mon Sep 17 00:00:00 2001 From: Denis Rouzaud Date: Tue, 9 Apr 2024 18:53:30 +0200 Subject: [PATCH] upgrade to psycopg3 (#218) --- .docker/Dockerfile | 4 ++-- datamodel/app/create_app.py | 4 ++-- datamodel/app/vw_sign_symbol.py | 17 ++++++++--------- scripts/all-signs.py | 11 +++++------ test/test_reordering.py | 5 ++--- test/utils.py | 13 ++++++------- 6 files changed, 25 insertions(+), 29 deletions(-) diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 59e3d040..2bafd3d4 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y python3 python3-pip python3-venv libpq- ENV VIRTUAL_ENV=/opt/venv RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN pip3 install pytest psycopg2 pirogue +RUN pip3 install pytest psycopg pirogue[psycopg3] # Add source ADD . /src @@ -37,7 +37,7 @@ RUN if [ "${AUTO_INIT}" = "True" ]; then ln -s /src/.docker/init_db.sh /docker-e # Some defaults ENV POSTGRES_PASSWORD=postgres -# otherwise psycopg2 cannot connect +# otherwise psycopg cannot connect ENV PGSERVICEFILE=/etc/postgresql-common/pg_service.conf ENV PGSERVICE=pg_signalo diff --git a/datamodel/app/create_app.py b/datamodel/app/create_app.py index fb75bfe5..763b33f3 100755 --- a/datamodel/app/create_app.py +++ b/datamodel/app/create_app.py @@ -4,7 +4,7 @@ import os import sys -import psycopg2 +import psycopg from pirogue import SimpleJoins from yaml import safe_load @@ -15,7 +15,7 @@ def run_sql(file_path: str, pg_service: str, variables: dict = {}): sql = open(file_path).read() - conn = psycopg2.connect(f"service={pg_service}") + conn = psycopg.connect(f"service={pg_service}") cursor = conn.cursor() cursor.execute(sql, variables) conn.commit() diff --git a/datamodel/app/vw_sign_symbol.py b/datamodel/app/vw_sign_symbol.py index abf8818d..fcfbe60e 100644 --- a/datamodel/app/vw_sign_symbol.py +++ b/datamodel/app/vw_sign_symbol.py @@ -5,7 +5,7 @@ import argparse import os -import psycopg2 +import psycopg from pirogue.utils import insert_command, select_columns, table_parts, update_command @@ -19,9 +19,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None): pg_service = os.getenv("PGSERVICE") assert pg_service - variables = {"SRID": int(srid)} - - conn = psycopg2.connect(f"service={pg_service}") + conn = psycopg.connect(f"service={pg_service}") cursor = conn.cursor() view_sql = """ @@ -41,7 +39,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None): , support.group_by_anchor , support.fk_support_type , support.fk_support_base_type - , support.geometry::geometry(Point,%(SRID)s) AS support_geometry + , support.geometry::geometry(Point,{srid}) AS support_geometry , COALESCE(vl_official_sign.directional_sign, vl_user_sign.directional_sign, FALSE) AS directional_sign , CASE WHEN sign.fk_sign_type = 15 THEN vl_user_sign.value_de @@ -197,7 +195,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None): , true::bool AS _verso , 1000 + ROW_NUMBER () OVER ( PARTITION BY support_id, jt.azimut ORDER BY frame_rank, sign_rank ) AS _rank FROM joined_tables jt - LEFT JOIN signalo_db.azimut az ON az.azimut = ((jt.azimut+180) %% 360) AND az.fk_support = support_id + LEFT JOIN signalo_db.azimut az ON az.azimut = ((jt.azimut+180) % 360) AND az.fk_support = support_id WHERE hanging_mode != 'RECTO'::signalo_db.sign_hanging AND group_by_anchor IS FALSE ORDER BY support_id, jt.azimut, _rank ), @@ -212,7 +210,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None): , true::bool AS _verso , 1000 + ROW_NUMBER () OVER ( PARTITION BY support_id, jt.azimut, frame_anchor ORDER BY frame_rank, sign_rank ) AS _rank FROM joined_tables jt - LEFT JOIN signalo_db.azimut az ON az.azimut = ((jt.azimut+180) %% 360) AND az.fk_support = support_id + LEFT JOIN signalo_db.azimut az ON az.azimut = ((jt.azimut+180) % 360) AND az.fk_support = support_id WHERE hanging_mode != 'RECTO'::signalo_db.sign_hanging AND group_by_anchor IS TRUE ORDER BY support_id, jt.azimut, frame_anchor, _rank ), @@ -281,6 +279,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None): END AS _img_direction FROM union_view uv; """.format( + srid=srid, sign_columns=select_columns( pg_cur=cursor, table_schema="signalo_db", @@ -309,8 +308,8 @@ def vw_sign_symbol(srid: int, pg_service: str = None): ) try: - cursor.execute(view_sql, variables) - except psycopg2.Error as e: + cursor.execute(view_sql) + except psycopg.Error as e: with open("~view.sql", "w") as f: f.write(view_sql) print(f"*** Failing:\n{view_sql}\n***") diff --git a/scripts/all-signs.py b/scripts/all-signs.py index 07cf86f5..b7fb2802 100755 --- a/scripts/all-signs.py +++ b/scripts/all-signs.py @@ -5,11 +5,10 @@ import math import os -import psycopg2 -import psycopg2.extras +import psycopg pg_service = os.environ.get("PGSERVICE") or "pg_signalo" -conn = psycopg2.connect(f"service={pg_service}") +conn = psycopg.connect(f"service={pg_service}") sign_per_support = 8 step = 100 # in meters @@ -19,7 +18,7 @@ def insert(table, row, schema="signalo_db"): cols = ", ".join(row.keys()) values = ", ".join([f"%({key})s" for key in row.keys()]) - cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = conn.cursor() cur.execute( "INSERT INTO {schema}.{table} ({cols}) VALUES ({values}) RETURNING id".format( table=table, schema=schema, cols=cols, values=values @@ -29,7 +28,7 @@ def insert(table, row, schema="signalo_db"): return cur.fetchone()[0] -cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) +cur = conn.cursor(row_factory=psycopg.rows.dict_row) cur.execute("SELECT * FROM signalo_db.vl_official_sign") rows = cur.fetchall() @@ -45,7 +44,7 @@ def insert(table, row, schema="signalo_db"): # create support + azimut + frame sql = f"SELECT ST_SetSRID(ST_MakePoint({2700000+x_shift*step}, {1300000+y_shift*step}), 2056);" - cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = conn.cursor() cur.execute(sql) geom = cur.fetchone()[0] diff --git a/test/test_reordering.py b/test/test_reordering.py index 03e1cdb3..f657c0dc 100644 --- a/test/test_reordering.py +++ b/test/test_reordering.py @@ -1,8 +1,7 @@ import os import unittest -import psycopg2 -import psycopg2.extras +import psycopg from .utils import DbTestBase @@ -19,7 +18,7 @@ def tearDown(cls) -> None: @classmethod def setUpClass(cls): pg_service = os.environ.get("PGSERVICE") or "signalo" - cls.conn = psycopg2.connect(f"service={pg_service}") + cls.conn = psycopg.connect(f"service={pg_service}") def test_reorder_signs_in_rank(self): frame_count = self.count("frame") diff --git a/test/utils.py b/test/utils.py index c7ce0b71..3ae2b67b 100644 --- a/test/utils.py +++ b/test/utils.py @@ -1,17 +1,16 @@ import unittest -import psycopg2 -import psycopg2.extras +import psycopg class DbTestBase: def count(self, table, schema="signalo_db") -> int: - cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = self.conn.cursor() cur.execute(f"SELECT COUNT(*) FROM {schema}.{table}") return cur.fetchone()[0] def select(self, table, id, schema="signalo_db"): - cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = self.conn.cursor(row_factory=psycopg.rows.dict_row) cur.execute( "SELECT * FROM {schema}.{table} WHERE id=%(id)s".format( table=table, schema=schema @@ -20,16 +19,16 @@ def select(self, table, id, schema="signalo_db"): ) return cur.fetchone() - def execute_select(self, sql: str, params=[]): + def execute_select(self, sql: str, params=None): return self.execute(f"SELECT {sql};", params=params).fetchone()[0] def execute(self, sql: str, params=[]): - cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur = self.conn.cursor() cur.execute(sql, params) return cur def cursor(self): - return self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor) + return self.conn.cursor(row_factory=psycopg.rows.dict_row) def insert(self, table, row, schema="signalo_db"): cur = self.conn.cursor()