From aaf85117e6f2dd3b28028d4cf809e4c2e58946ef Mon Sep 17 00:00:00 2001 From: mgcam Date: Fri, 2 Feb 2024 15:29:19 +0000 Subject: [PATCH] Made 'properties' column nullable. The properties column of the LangQC sub_product database table is made nullable. The correctness of data in this column is no longer guaranteed, the column will be dropped in future. --- CHANGELOG.md | 11 ++++++++ alembic/README.md | 2 +- ...03a709a_make_column_properties_nullable.py | 28 +++++++++++++++++++ lang_qc/db/helper/well.py | 5 ---- lang_qc/db/qc_schema.py | 2 +- pyproject.toml | 2 +- tests/fixtures/inbox_data.py | 2 -- tests/fixtures/well_data.py | 3 -- 8 files changed, 42 insertions(+), 13 deletions(-) create mode 100644 alembic/versions/3814003a709a_make_column_properties_nullable.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ec5e7cd8..748e26fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] +### Changed + +* The properties column of the LangQC sub_product database table + is made nullable. The correctness of data in this column is no + longer guaranteed, the column will be dropped in future. + +* The production code no longer depends on the npg_id_generation + package, therefore this dependency was moved to the dev section + of pyproject.toml. The package is still used by tests and test + fixtures. + ## [1.5.1] - 2024-01-25 ### Changed diff --git a/alembic/README.md b/alembic/README.md index 8d5bcd8a..e70d896a 100644 --- a/alembic/README.md +++ b/alembic/README.md @@ -5,7 +5,7 @@ Planned changes should be applied in [lang_qc/db/qc_schema.py](../lang_qc/db/qc_schema.py). We recommend doing this manually. -Changes to unit test,test fixtures and the source code should be applied as +Changes to unit tests, test fixtures and the source code should be applied as appropriate. ## Creating migrations for the QC database schema diff --git a/alembic/versions/3814003a709a_make_column_properties_nullable.py b/alembic/versions/3814003a709a_make_column_properties_nullable.py new file mode 100644 index 00000000..20d1f244 --- /dev/null +++ b/alembic/versions/3814003a709a_make_column_properties_nullable.py @@ -0,0 +1,28 @@ +"""Make column properties nullable + +Revision ID: 3814003a709a +Revises: 36952df5b8ba +Create Date: 2024-02-02 14:30:20.079955 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "3814003a709a" +down_revision = "36952df5b8ba" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + sql = """ + ALTER TABLE sub_product MODIFY properties JSON DEFAULT NULL + """ + op.execute(sql) + + +def downgrade() -> None: + sql = """ + ALTER TABLE sub_product MODIFY properties JSON NOT NULL + """ + op.execute(sql) diff --git a/lang_qc/db/helper/well.py b/lang_qc/db/helper/well.py index 6879f1a4..66f27f7d 100644 --- a/lang_qc/db/helper/well.py +++ b/lang_qc/db/helper/well.py @@ -24,7 +24,6 @@ for interaction with the LangQC database. """ -from npg_id_generation.pac_bio import PacBioEntity from sqlalchemy import select from sqlalchemy.orm import Session @@ -84,9 +83,6 @@ def _create_well( product_attr_pn = session.execute( select(SubProductAttr).where(SubProductAttr.attr_name == "plate_number") ).scalar_one() - product_json = PacBioEntity( - run_name=run_name, well_label=well_label, plate_number=plate_number - ).model_dump_json() # TODO: in future for composite products we have to check whether any of # the `sub_product` table entries we are linking to already exist. @@ -103,7 +99,6 @@ def _create_well( value_attr_three=str(plate_number) if plate_number is not None else None, - properties=product_json, properties_digest=id_product, ) ], diff --git a/lang_qc/db/qc_schema.py b/lang_qc/db/qc_schema.py index bf6506ce..75bb5d24 100644 --- a/lang_qc/db/qc_schema.py +++ b/lang_qc/db/qc_schema.py @@ -180,7 +180,7 @@ class SubProduct(Base): value_attr_two = Column(String(20), nullable=False, index=True) id_attr_three = Column(INTEGER, nullable=True, index=True) value_attr_three = Column(String(20), nullable=True, index=True) - properties = Column(JSON, nullable=False) + properties = Column(JSON, nullable=True) properties_digest = Column(CHAR(64), nullable=False, unique=True) tags = Column(String(255), index=True) diff --git a/pyproject.toml b/pyproject.toml index bc0d22b3..c299c5b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,9 +16,9 @@ cryptography = { version = "^40.0.2" } SQLAlchemy = { version = "^2.0.1", extras = ["pymysql"] } pydantic = "^2.4" pydantic-settings = "^2.0" -npg_id_generation = { git = "https://github.com/wtsi-npg/npg_id_generation.git", tag="4.0.1" } [tool.poetry.dev-dependencies] +npg_id_generation = { git = "https://github.com/wtsi-npg/npg_id_generation.git", tag="4.0.1" } black = "^22.3.0" flake8 = "^4.0.1" pytest = "^7.1.1" diff --git a/tests/fixtures/inbox_data.py b/tests/fixtures/inbox_data.py index 92253113..b8fb04e8 100644 --- a/tests/fixtures/inbox_data.py +++ b/tests/fixtures/inbox_data.py @@ -170,7 +170,6 @@ def setup_data(desired_wells): mlwhdb_test_session.add(run_metrics) if state is not None: - json = pbe.model_dump_json() qc_state = QcState( created_by="me", @@ -186,7 +185,6 @@ def setup_data(desired_wells): sub_product_attr_=well_label_attr, value_attr_one=run_name, value_attr_two=well_label, - properties=json, properties_digest=id, ), ], diff --git a/tests/fixtures/well_data.py b/tests/fixtures/well_data.py index 68218567..abb99a36 100644 --- a/tests/fixtures/well_data.py +++ b/tests/fixtures/well_data.py @@ -845,7 +845,6 @@ def load_data4well_retrieval( run_name=qc_data[0], well_label=qc_data[1], plate_number=qc_data[5] ) id_product = pbe.hash_product_id() - json = pbe.model_dump_json() date = datetime.strptime(qc_data[4], DATE_FORMAT) seq_product = SeqProduct( @@ -859,7 +858,6 @@ def load_data4well_retrieval( value_attr_one=qc_data[0], value_attr_two=qc_data[1], value_attr_three=qc_data[5], - properties=json, properties_digest=id_product, ), ], @@ -955,7 +953,6 @@ def load_data4qc_assign(load_dicts_and_users, qcdb_test_session): value_attr_one="TRACTION_RUN_2", value_attr_two=label, value_attr_three="2", - properties=p.model_dump_json(), properties_digest=id, ), ],