upgrade to psycopg3 (#218)

.docker/Dockerfile

@@ -15,7 +15,7 @@ RUN apt-get update && apt-get install -y python3 python3-pip python3-venv libpq-
 ENV VIRTUAL_ENV=/opt/venv
 RUN python3 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN pip3 install pytest psycopg2 pirogue
+RUN pip3 install pytest psycopg pirogue[psycopg3]
 
 # Add source
 ADD . /src
@@ -37,7 +37,7 @@ RUN if [ "${AUTO_INIT}" = "True" ]; then ln -s /src/.docker/init_db.sh /docker-e
 
 # Some defaults
 ENV POSTGRES_PASSWORD=postgres
-# otherwise psycopg2 cannot connect
+# otherwise psycopg cannot connect
 ENV PGSERVICEFILE=/etc/postgresql-common/pg_service.conf
 
 ENV PGSERVICE=pg_signalo

datamodel/app/create_app.py

@@ -4,7 +4,7 @@
 import os
 import sys
 
-import psycopg2
+import psycopg
 from pirogue import SimpleJoins
 from yaml import safe_load
 
@@ -15,7 +15,7 @@
 
 def run_sql(file_path: str, pg_service: str, variables: dict = {}):
     sql = open(file_path).read()
-    conn = psycopg2.connect(f"service={pg_service}")
+    conn = psycopg.connect(f"service={pg_service}")
     cursor = conn.cursor()
     cursor.execute(sql, variables)
     conn.commit()

datamodel/app/vw_sign_symbol.py

@@ -5,7 +5,7 @@
 import argparse
 import os
 
-import psycopg2
+import psycopg
 from pirogue.utils import insert_command, select_columns, table_parts, update_command
 
 
@@ -19,9 +19,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None):
         pg_service = os.getenv("PGSERVICE")
     assert pg_service
 
-    variables = {"SRID": int(srid)}
-
-    conn = psycopg2.connect(f"service={pg_service}")
+    conn = psycopg.connect(f"service={pg_service}")
     cursor = conn.cursor()
 
     view_sql = """
@@ -41,7 +39,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None):
                 , support.group_by_anchor
                 , support.fk_support_type
                 , support.fk_support_base_type
-                , support.geometry::geometry(Point,%(SRID)s) AS support_geometry
+                , support.geometry::geometry(Point,{srid}) AS support_geometry
                 , COALESCE(vl_official_sign.directional_sign, vl_user_sign.directional_sign, FALSE) AS directional_sign
                 , CASE
                     WHEN sign.fk_sign_type = 15 THEN vl_user_sign.value_de
@@ -197,7 +195,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None):
                 , true::bool AS _verso
                 , 1000 + ROW_NUMBER () OVER ( PARTITION BY support_id, jt.azimut ORDER BY frame_rank, sign_rank ) AS _rank
             FROM joined_tables jt
-            LEFT JOIN signalo_db.azimut az ON az.azimut = ((jt.azimut+180) %% 360) AND az.fk_support = support_id
+            LEFT JOIN signalo_db.azimut az ON az.azimut = ((jt.azimut+180) % 360) AND az.fk_support = support_id
             WHERE hanging_mode != 'RECTO'::signalo_db.sign_hanging AND group_by_anchor IS FALSE
             ORDER BY support_id, jt.azimut, _rank
         ),
@@ -212,7 +210,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None):
                 , true::bool AS _verso
                 , 1000 + ROW_NUMBER () OVER ( PARTITION BY support_id, jt.azimut, frame_anchor ORDER BY frame_rank, sign_rank ) AS _rank
             FROM joined_tables jt
-            LEFT JOIN signalo_db.azimut az ON az.azimut = ((jt.azimut+180) %% 360) AND az.fk_support = support_id
+            LEFT JOIN signalo_db.azimut az ON az.azimut = ((jt.azimut+180) % 360) AND az.fk_support = support_id
             WHERE hanging_mode != 'RECTO'::signalo_db.sign_hanging AND group_by_anchor IS TRUE
             ORDER BY support_id, jt.azimut, frame_anchor, _rank
         ),
@@ -281,6 +279,7 @@ def vw_sign_symbol(srid: int, pg_service: str = None):
                   END AS _img_direction
             FROM union_view uv;
     """.format(
+        srid=srid,
         sign_columns=select_columns(
             pg_cur=cursor,
             table_schema="signalo_db",
@@ -309,8 +308,8 @@ def vw_sign_symbol(srid: int, pg_service: str = None):
     )
 
     try:
-        cursor.execute(view_sql, variables)
-    except psycopg2.Error as e:
+        cursor.execute(view_sql)
+    except psycopg.Error as e:
         with open("~view.sql", "w") as f:
             f.write(view_sql)
         print(f"*** Failing:\n{view_sql}\n***")

scripts/all-signs.py

@@ -5,11 +5,10 @@
 import math
 import os
 
-import psycopg2
-import psycopg2.extras
+import psycopg
 
 pg_service = os.environ.get("PGSERVICE") or "pg_signalo"
-conn = psycopg2.connect(f"service={pg_service}")
+conn = psycopg.connect(f"service={pg_service}")
 
 sign_per_support = 8
 step = 100  # in meters
@@ -19,7 +18,7 @@
 def insert(table, row, schema="signalo_db"):
     cols = ", ".join(row.keys())
     values = ", ".join([f"%({key})s" for key in row.keys()])
-    cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+    cur = conn.cursor()
     cur.execute(
         "INSERT INTO {schema}.{table} ({cols}) VALUES ({values}) RETURNING id".format(
             table=table, schema=schema, cols=cols, values=values
@@ -29,7 +28,7 @@ def insert(table, row, schema="signalo_db"):
     return cur.fetchone()[0]
 
 
-cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+cur = conn.cursor(row_factory=psycopg.rows.dict_row)
 cur.execute("SELECT * FROM signalo_db.vl_official_sign")
 rows = cur.fetchall()
 
@@ -45,7 +44,7 @@ def insert(table, row, schema="signalo_db"):
 
         # create support + azimut + frame
         sql = f"SELECT ST_SetSRID(ST_MakePoint({2700000+x_shift*step}, {1300000+y_shift*step}), 2056);"
-        cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        cur = conn.cursor()
         cur.execute(sql)
         geom = cur.fetchone()[0]
 

test/test_reordering.py

@@ -1,8 +1,7 @@
 import os
 import unittest
 
-import psycopg2
-import psycopg2.extras
+import psycopg
 
 from .utils import DbTestBase
 
@@ -19,7 +18,7 @@ def tearDown(cls) -> None:
     @classmethod
     def setUpClass(cls):
         pg_service = os.environ.get("PGSERVICE") or "signalo"
-        cls.conn = psycopg2.connect(f"service={pg_service}")
+        cls.conn = psycopg.connect(f"service={pg_service}")
 
     def test_reorder_signs_in_rank(self):
         frame_count = self.count("frame")

test/utils.py

@@ -1,17 +1,16 @@
 import unittest
 
-import psycopg2
-import psycopg2.extras
+import psycopg
 
 
 class DbTestBase:
     def count(self, table, schema="signalo_db") -> int:
-        cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        cur = self.conn.cursor()
         cur.execute(f"SELECT COUNT(*) FROM {schema}.{table}")
         return cur.fetchone()[0]
 
     def select(self, table, id, schema="signalo_db"):
-        cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        cur = self.conn.cursor(row_factory=psycopg.rows.dict_row)
         cur.execute(
             "SELECT * FROM {schema}.{table} WHERE id=%(id)s".format(
                 table=table, schema=schema
@@ -20,16 +19,16 @@ def select(self, table, id, schema="signalo_db"):
         )
         return cur.fetchone()
 
-    def execute_select(self, sql: str, params=[]):
+    def execute_select(self, sql: str, params=None):
         return self.execute(f"SELECT {sql};", params=params).fetchone()[0]
 
     def execute(self, sql: str, params=[]):
-        cur = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        cur = self.conn.cursor()
         cur.execute(sql, params)
         return cur
 
     def cursor(self):
-        return self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
+        return self.conn.cursor(row_factory=psycopg.rows.dict_row)
 
     def insert(self, table, row, schema="signalo_db"):
         cur = self.conn.cursor()