diff --git a/dandiapi/zarr/migrations/0005_zarr_checksum_nonnull.py b/dandiapi/zarr/migrations/0005_zarr_checksum_nonnull.py new file mode 100644 index 000000000..3e219d8c0 --- /dev/null +++ b/dandiapi/zarr/migrations/0005_zarr_checksum_nonnull.py @@ -0,0 +1,62 @@ +# Generated by Django 4.1.13 on 2023-11-09 00:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ( + 'zarr', + '0004_remove_embargoedzarrarchive_zarr-embargoedzarrarchive-' + 'consistent-checksum-status_and_more', + ), + ] + + operations = [ + migrations.RemoveConstraint( + model_name='embargoedzarrarchive', + name='zarr-embargoedzarrarchive-consistent-checksum-status', + ), + migrations.RemoveConstraint( + model_name='zarrarchive', + name='zarr-zarrarchive-consistent-checksum-status', + ), + migrations.AlterField( + model_name='embargoedzarrarchive', + name='checksum', + field=models.CharField(default='', max_length=512), + preserve_default=False, + ), + migrations.AlterField( + model_name='zarrarchive', + name='checksum', + field=models.CharField(default='', max_length=512), + preserve_default=False, + ), + migrations.AddConstraint( + model_name='embargoedzarrarchive', + constraint=models.CheckConstraint( + check=models.Q( + models.Q( + ('checksum', ''), ('status__in', ['Pending', 'Uploaded', 'Ingesting']) + ), + models.Q(models.Q(('checksum', ''), _negated=True), ('status', 'Complete')), + _connector='OR', + ), + name='zarr-embargoedzarrarchive-consistent-checksum-status', + ), + ), + migrations.AddConstraint( + model_name='zarrarchive', + constraint=models.CheckConstraint( + check=models.Q( + models.Q( + ('checksum', ''), ('status__in', ['Pending', 'Uploaded', 'Ingesting']) + ), + models.Q(models.Q(('checksum', ''), _negated=True), ('status', 'Complete')), + _connector='OR', + ), + name='zarr-zarrarchive-consistent-checksum-status', + ), + ), + ] diff --git a/dandiapi/zarr/models.py b/dandiapi/zarr/models.py index e7173ff02..e72e4eccd 100644 --- a/dandiapi/zarr/models.py +++ b/dandiapi/zarr/models.py @@ -39,14 +39,14 @@ class Meta: models.CheckConstraint( name='%(app_label)s-%(class)s-consistent-checksum-status', check=models.Q( - checksum__isnull=True, + checksum='', status__in=[ ZarrArchiveStatus.PENDING, ZarrArchiveStatus.UPLOADED, ZarrArchiveStatus.INGESTING, ], ) - | models.Q(checksum__isnull=False, status=ZarrArchiveStatus.COMPLETE), + | (~models.Q(checksum='') & models.Q(status=ZarrArchiveStatus.COMPLETE)), ), ] @@ -54,7 +54,7 @@ class Meta: name = models.CharField(max_length=512) file_count = models.BigIntegerField(default=0) size = models.BigIntegerField(default=0) - checksum = models.CharField(max_length=512, null=True, default=None) + checksum = models.CharField(max_length=512) status = models.CharField( max_length=max(len(choice[0]) for choice in ZarrArchiveStatus.choices), choices=ZarrArchiveStatus.choices, @@ -80,7 +80,7 @@ def generate_upload_urls(self, path_md5s: list[dict]): ] def mark_pending(self): - self.checksum = None + self.checksum = '' self.status = ZarrArchiveStatus.PENDING self.file_count = 0 self.size = 0 diff --git a/dandiapi/zarr/tasks/__init__.py b/dandiapi/zarr/tasks/__init__.py index 81bbd921a..43b9be9fe 100644 --- a/dandiapi/zarr/tasks/__init__.py +++ b/dandiapi/zarr/tasks/__init__.py @@ -24,7 +24,7 @@ def ingest_zarr_archive(zarr_id: str, force: bool = False): # Set as ingesting zarr.status = ZarrArchiveStatus.INGESTING - zarr.checksum = None + zarr.checksum = '' zarr.save(update_fields=['status', 'checksum']) # Zarr is in correct state, lock until ingestion finishes diff --git a/dandiapi/zarr/tests/test_ingest_zarr_archive.py b/dandiapi/zarr/tests/test_ingest_zarr_archive.py index 38f476ce6..c9e20f6e7 100644 --- a/dandiapi/zarr/tests/test_ingest_zarr_archive.py +++ b/dandiapi/zarr/tests/test_ingest_zarr_archive.py @@ -22,7 +22,7 @@ def test_ingest_zarr_archive(zarr_archive_factory, zarr_file_factory): total_size = sum([f.size for f in files]) # Assert pre-ingest properties - assert zarr.checksum is None + assert zarr.checksum == '' assert zarr.size == 0 assert zarr.file_count == 0 assert zarr.status == ZarrArchiveStatus.UPLOADED @@ -32,7 +32,7 @@ def test_ingest_zarr_archive(zarr_archive_factory, zarr_file_factory): # Assert checksum properly computed zarr.refresh_from_db() - assert zarr.checksum is not None + assert zarr.checksum != '' assert zarr.size == total_size assert zarr.file_count == 2 assert zarr.status == ZarrArchiveStatus.COMPLETE @@ -76,7 +76,7 @@ def test_ingest_zarr_archive_force(zarr_archive_factory, zarr_file_factory): # Perform ingest with force flag, assert updated ingest_zarr_archive(str(zarr.zarr_id), force=True) zarr.refresh_from_db() - assert zarr.checksum is not None and zarr.checksum != first_checksum + assert zarr.checksum != '' and zarr.checksum != first_checksum @pytest.mark.django_db(transaction=True) @@ -88,7 +88,7 @@ def test_ingest_zarr_archive_assets(zarr_archive_factory, zarr_file_factory, dra # Assert asset size, metadata assert asset.size == 0 assert asset.full_metadata['contentSize'] == 0 - assert asset.full_metadata['digest']['dandi:dandi-zarr-checksum'] is None + assert asset.full_metadata['digest']['dandi:dandi-zarr-checksum'] == '' # Compute checksum zarr_file = zarr_file_factory(zarr_archive=zarr) @@ -158,4 +158,4 @@ def test_ingest_dandiset_zarrs(dandiset_factory, zarr_archive_factory, zarr_file for zarr in dandiset.zarr_archives.all(): assert zarr.size != 0 assert zarr.file_count != 0 - assert zarr.checksum is not None + assert zarr.checksum != '' diff --git a/dandiapi/zarr/tests/test_zarr.py b/dandiapi/zarr/tests/test_zarr.py index 0f2444f38..a0537e123 100644 --- a/dandiapi/zarr/tests/test_zarr.py +++ b/dandiapi/zarr/tests/test_zarr.py @@ -27,7 +27,7 @@ def test_zarr_rest_create(authenticated_api_client, user, dandiset): 'zarr_id': UUID_RE, 'dandiset': dandiset.identifier, 'status': ZarrArchiveStatus.PENDING, - 'checksum': None, + 'checksum': '', 'file_count': 0, 'size': 0, } @@ -240,7 +240,7 @@ def test_zarr_rest_delete_file( # Assert zarr is back in pending state zarr_archive.refresh_from_db() assert zarr_archive.status == ZarrArchiveStatus.PENDING - assert zarr_archive.checksum is None + assert zarr_archive.checksum == '' assert zarr_archive.file_count == 0 assert zarr_archive.size == 0 diff --git a/dandiapi/zarr/tests/test_zarr_upload.py b/dandiapi/zarr/tests/test_zarr_upload.py index 11fee8f8a..0c64b6d42 100644 --- a/dandiapi/zarr/tests/test_zarr_upload.py +++ b/dandiapi/zarr/tests/test_zarr_upload.py @@ -34,7 +34,7 @@ def test_zarr_rest_upload_start( # Assert fields updated zarr_archive.refresh_from_db() assert zarr_archive.status == ZarrArchiveStatus.PENDING - assert zarr_archive.checksum is None + assert zarr_archive.checksum == '' assert zarr_archive.file_count == 0 assert zarr_archive.size == 0 @@ -80,7 +80,7 @@ def test_zarr_rest_finalize( # Check that zarr ingestion occurred zarr_archive.refresh_from_db() - assert zarr_archive.checksum is not None + assert zarr_archive.checksum != '' assert zarr_archive.checksum != EMPTY_CHECKSUM assert zarr_archive.status == ZarrArchiveStatus.COMPLETE