Merge pull request #194 from mgcam/stop_saving_json

Made 'properties' column nullable.

CHANGELOG.md

@@ -5,6 +5,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Changed
+
+* The properties column of the LangQC sub_product database table
+  is made nullable. The correctness of data in this column is no
+  longer guaranteed, the column will be dropped in future.
+
+* The production code no longer depends on the npg_id_generation
+  package, therefore this dependency was moved to the dev section
+  of pyproject.toml. The package is still used by tests and test
+  fixtures.
+
 ## [1.5.1] - 2024-01-25
 
 ### Changed

alembic/README.md

@@ -5,7 +5,7 @@
 Planned changes should be applied in [lang_qc/db/qc_schema.py](../lang_qc/db/qc_schema.py).
 We recommend doing this manually.
 
-Changes to unit test,test fixtures and the source code should be applied as
+Changes to unit tests, test fixtures and the source code should be applied as
 appropriate. 
 
 ## Creating migrations for the QC database schema

alembic/versions/3814003a709a_make_column_properties_nullable.py

@@ -0,0 +1,28 @@
+"""Make column properties nullable
+
+Revision ID: 3814003a709a
+Revises: 36952df5b8ba
+Create Date: 2024-02-02 14:30:20.079955
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3814003a709a"
+down_revision = "36952df5b8ba"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    sql = """
+    ALTER TABLE sub_product MODIFY properties JSON DEFAULT NULL
+    """
+    op.execute(sql)
+
+
+def downgrade() -> None:
+    sql = """
+    ALTER TABLE sub_product MODIFY properties JSON NOT NULL
+    """
+    op.execute(sql)

lang_qc/db/helper/well.py

@@ -24,7 +24,6 @@
 for interaction with the LangQC database.
 """
 
-from npg_id_generation.pac_bio import PacBioEntity
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
@@ -84,9 +83,6 @@ def _create_well(
     product_attr_pn = session.execute(
         select(SubProductAttr).where(SubProductAttr.attr_name == "plate_number")
     ).scalar_one()
-    product_json = PacBioEntity(
-        run_name=run_name, well_label=well_label, plate_number=plate_number
-    ).model_dump_json()
 
     # TODO: in future for composite products we have to check whether any of
     # the `sub_product` table entries we are linking to already exist.
@@ -103,7 +99,6 @@ def _create_well(
                 value_attr_three=str(plate_number)
                 if plate_number is not None
                 else None,
-                properties=product_json,
                 properties_digest=id_product,
             )
         ],

lang_qc/db/qc_schema.py

@@ -180,7 +180,7 @@ class SubProduct(Base):
     value_attr_two = Column(String(20), nullable=False, index=True)
     id_attr_three = Column(INTEGER, nullable=True, index=True)
     value_attr_three = Column(String(20), nullable=True, index=True)
-    properties = Column(JSON, nullable=False)
+    properties = Column(JSON, nullable=True)
     properties_digest = Column(CHAR(64), nullable=False, unique=True)
     tags = Column(String(255), index=True)
 

pyproject.toml

@@ -16,9 +16,9 @@ cryptography = { version = "^40.0.2" }
 SQLAlchemy = { version = "^2.0.1", extras = ["pymysql"] }
 pydantic = "^2.4"
 pydantic-settings = "^2.0"
-npg_id_generation = { git = "https://github.com/wtsi-npg/npg_id_generation.git", tag="4.0.1" }
 
 [tool.poetry.dev-dependencies]
+npg_id_generation = { git = "https://github.com/wtsi-npg/npg_id_generation.git", tag="4.0.1" }
 black = "^22.3.0"
 flake8 = "^4.0.1"
 pytest = "^7.1.1"

tests/fixtures/inbox_data.py

@@ -170,7 +170,6 @@ def setup_data(desired_wells):
                 mlwhdb_test_session.add(run_metrics)
 
                 if state is not None:
-                    json = pbe.model_dump_json()
 
                     qc_state = QcState(
                         created_by="me",
@@ -186,7 +185,6 @@ def setup_data(desired_wells):
                                     sub_product_attr_=well_label_attr,
                                     value_attr_one=run_name,
                                     value_attr_two=well_label,
-                                    properties=json,
                                     properties_digest=id,
                                 ),
                             ],

tests/fixtures/well_data.py

@@ -845,7 +845,6 @@ def load_data4well_retrieval(
             run_name=qc_data[0], well_label=qc_data[1], plate_number=qc_data[5]
         )
         id_product = pbe.hash_product_id()
-        json = pbe.model_dump_json()
         date = datetime.strptime(qc_data[4], DATE_FORMAT)
 
         seq_product = SeqProduct(
@@ -859,7 +858,6 @@ def load_data4well_retrieval(
                     value_attr_one=qc_data[0],
                     value_attr_two=qc_data[1],
                     value_attr_three=qc_data[5],
-                    properties=json,
                     properties_digest=id_product,
                 ),
             ],
@@ -955,7 +953,6 @@ def load_data4qc_assign(load_dicts_and_users, qcdb_test_session):
                     value_attr_one="TRACTION_RUN_2",
                     value_attr_two=label,
                     value_attr_three="2",
-                    properties=p.model_dump_json(),
                     properties_digest=id,
                 ),
             ],