Merge pull request #133 from wtsi-npg/devel

merge from devel to master to create release 6.4.0
wtsi-npg · Oct 31, 2019 · 04f8642 · 04f8642
2 parents c2c0902 + fde8d5a
commit 04f8642
Show file tree

Hide file tree

Showing 7 changed files with 296 additions and 108 deletions.
diff --git a/Build.PL b/Build.PL
@@ -22,6 +22,7 @@ my $builder = WTSI::DNAP::Utilities::Build->new(
     'SQL::Translator'          => 0,
     'Test::Distribution'       => 0,
     'Test::Exception'          => 0,
+    'Test::Warn'               => 0,
     'Test::More'               => 0,
     'Test::Perl::Critic'       => 0,
     'Test::Pod'                => 0,

diff --git a/Changes b/Changes
@@ -1,6 +1,18 @@
 LIST OF CHANGES
 ---------------
 
+release 6.4.0
+ - iseq_external_product_component table redesign to allow for
+   rows on iseq_product_metrics table to be deleted and recreated
+   without making data in the linking table invalid
+ - yield column in the iseq_external_product_component table changed
+   from integer to float to correctly accommodate source data
+ - code to creating rows in the iseq_external_product_component table,
+   which will be run on row creation the and sometimes on row update
+   for rows of the iseq_external_product_metrics table; this code will
+   also be used to create a backlog of linking rows for existing
+   iseq_external_product_metrics table rows
+
 release 6.3.0
  - table update for iseq_external_product_metrics:
    drops id_iseq_pr_metrics_tmp and a foreign key based on it

diff --git a/lib/WTSI/DNAP/Warehouse/Schema/Result/IseqExternalProductComponent.pm b/lib/WTSI/DNAP/Warehouse/Schema/Result/IseqExternalProductComponent.pm
@@ -10,7 +10,7 @@ WTSI::DNAP::Warehouse::Schema::Result::IseqExternalProductComponent
 
 =head1 DESCRIPTION
 
-Table linking iseq_external_product_metrics table data to components in the iseq_product_metrics table
+Table linking iseq_external_product_metrics table products to components in the iseq_product_metrics table
 
 =cut
 
@@ -51,23 +51,22 @@ __PACKAGE__->table('iseq_external_product_components');
 
 Internal to this database id, value can change
 
-=head2 id_iseq_pr_tmp
+=head2 id_iseq_product_ext
 
-  data_type: 'bigint'
-  extra: {unsigned => 1}
+  data_type: 'char'
   is_foreign_key: 1
   is_nullable: 0
+  size: 64
 
-iseq_external_product_metrics table row id for the product
+id (digest) for the external product composition
 
-=head2 id_iseq_pr_component_tmp
+=head2 id_iseq_product
 
-  data_type: 'bigint'
-  extra: {unsigned => 1}
-  is_foreign_key: 1
-  is_nullable: 1
+  data_type: 'char'
+  is_nullable: 0
+  size: 64
 
-iseq_product_metrics table row id for one of this product's components
+id (digest) for one of the products components
 
 =head2 num_components
 
@@ -95,20 +94,10 @@ __PACKAGE__->add_columns(
     is_auto_increment => 1,
     is_nullable => 0,
   },
-  'id_iseq_pr_tmp',
-  {
-    data_type => 'bigint',
-    extra => { unsigned => 1 },
-    is_foreign_key => 1,
-    is_nullable => 0,
-  },
-  'id_iseq_pr_component_tmp',
-  {
-    data_type => 'bigint',
-    extra => { unsigned => 1 },
-    is_foreign_key => 1,
-    is_nullable => 1,
-  },
+  'id_iseq_product_ext',
+  { data_type => 'char', is_foreign_key => 1, is_nullable => 0, size => 64 },
+  'id_iseq_product',
+  { data_type => 'char', is_nullable => 0, size => 64 },
   'num_components',
   { data_type => 'tinyint', extra => { unsigned => 1 }, is_nullable => 0 },
   'component_index',
@@ -133,22 +122,22 @@ __PACKAGE__->set_primary_key('id_iseq_ext_pr_components_tmp');
 
 =over 4
 
-=item * L</id_iseq_pr_tmp>
+=item * L</id_iseq_product>
 
-=item * L</id_iseq_pr_component_tmp>
+=item * L</id_iseq_product_ext>
 
 =back
 
 =cut
 
 __PACKAGE__->add_unique_constraint(
   'iseq_ext_pr_comp_unique',
-  ['id_iseq_pr_tmp', 'id_iseq_pr_component_tmp'],
+  ['id_iseq_product', 'id_iseq_product_ext'],
 );
 
 =head1 RELATIONS
 
-=head2 iseq_product
+=head2 iseq_product_ext
 
 Type: belongs_to
 
@@ -157,13 +146,19 @@ Related object: L<WTSI::DNAP::Warehouse::Schema::Result::IseqExternalProductMetr
 =cut
 
 __PACKAGE__->belongs_to(
-  'iseq_product',
+  'iseq_product_ext',
   'WTSI::DNAP::Warehouse::Schema::Result::IseqExternalProductMetric',
-  { id_iseq_ext_pr_metrics_tmp => 'id_iseq_pr_tmp' },
-  { is_deferrable => 1, on_delete => 'CASCADE', on_update => 'NO ACTION' },
+  { id_iseq_product => 'id_iseq_product_ext' },
+  { is_deferrable => 1, on_delete => 'NO ACTION', on_update => 'NO ACTION' },
 );
 
-=head2 iseq_product_component
+
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-10-18 16:31:06
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:MZF9h1iwW2028nqp+S3KRA
+
+our $VERSION = '0';
+
+=head2 iseq_product
 
 Type: belongs_to
 
@@ -172,23 +167,12 @@ Related object: L<WTSI::DNAP::Warehouse::Schema::Result::IseqProductMetric>
 =cut
 
 __PACKAGE__->belongs_to(
-  'iseq_product_component',
+  'iseq_product',
   'WTSI::DNAP::Warehouse::Schema::Result::IseqProductMetric',
-  { id_iseq_pr_metrics_tmp => 'id_iseq_pr_component_tmp' },
-  {
-    is_deferrable => 1,
-    join_type     => 'LEFT',
-    on_delete     => 'SET NULL',
-    on_update     => 'NO ACTION',
-  },
+  { id_iseq_product => 'id_iseq_product' },
+  { is_deferrable => 1, on_delete => 'NO ACTION', on_update => 'NO ACTION' },
 );
 
-
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-09-10 13:54:06
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:zj/Db+JD6MS1y/fGCLjDcA
-
-our $VERSION = '0';
-
 __PACKAGE__->meta->make_immutable;
 
 1;

diff --git a/lib/WTSI/DNAP/Warehouse/Schema/Result/IseqExternalProductMetric.pm b/lib/WTSI/DNAP/Warehouse/Schema/Result/IseqExternalProductMetric.pm
@@ -366,11 +366,11 @@ Fraction of marker pairs with two read pairs evidencing parity and non-parity, m
 
 =head2 yield
 
-  data_type: 'smallint'
+  data_type: 'float'
   extra: {unsigned => 1}
   is_nullable: 1
 
-sequence data quantity (Gb)
+sequence data quantity (Gb), autosome
 
 =head2 yield_q20
 
@@ -692,7 +692,7 @@ __PACKAGE__->add_columns(
   'contamination_assessment',
   { data_type => 'char', is_nullable => 1, size => 4 },
   'yield',
-  { data_type => 'smallint', extra => { unsigned => 1 }, is_nullable => 1 },
+  { data_type => 'float', extra => { unsigned => 1 }, is_nullable => 1 },
   'yield_q20',
   { data_type => 'bigint', extra => { unsigned => 1 }, is_nullable => 1 },
   'yield_q30',
@@ -790,13 +790,13 @@ Related object: L<WTSI::DNAP::Warehouse::Schema::Result::IseqExternalProductComp
 __PACKAGE__->has_many(
   'iseq_external_product_components',
   'WTSI::DNAP::Warehouse::Schema::Result::IseqExternalProductComponent',
-  { 'foreign.id_iseq_pr_tmp' => 'self.id_iseq_ext_pr_metrics_tmp' },
+  { 'foreign.id_iseq_product_ext' => 'self.id_iseq_product' },
   { cascade_copy => 0, cascade_delete => 0 },
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-10-15 12:49:23
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:tw/Esr1WGdB82BXF24C2Fg
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-10-31 15:09:17
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:bZN6W3/2xfdBmBWBAS2Ybw
 
 use Readonly;
 use Try::Tiny;
@@ -818,18 +818,31 @@ our $VERSION = '0';
 around [qw/update insert/] => sub {
   my $orig = shift;
   my $self = shift;
+
+  my $composition;
   if (not $self->in_storage # this is an insert
       or not $self->id_iseq_product) {
     try {
       my %meta = %{$self->file_name2meta()};
+      $composition = delete $meta{'composition'};
       while (my ($column_name, $value) = each %meta) {
         $self->$column_name($value);
       }
     } catch {
-      carp $_;
+      carp 'Warning updating or creating a row: ' . $_;
+    };
+  }
+  my $row = $self->$orig(@_); # create or update row
+
+  if ($composition) {
+    try {
+      $self->create_component_linking_rows($composition);
+    } catch {
+      carp 'Warning updating or creating a linking row: ' . $_;
     };
   }
-  return $self->$orig(@_);
+
+  return $row;
 };
 
 sub file_name2composition {
@@ -870,10 +883,8 @@ sub file_name2composition {
       @composition_jsons == 1 or croak
         "Multiple merged products found for '$name'";
 
-      foreach my $component (npg_tracking::glossary::composition->thaw(
-        $composition_jsons[0],
-        component_class => 'npg_tracking::glossary::composition::component::illumina'
-      )->components_list) {
+      foreach my $component (_json2composition(
+          $composition_jsons[0])->components_list) {
         $factory->add_component($component);
       }
     }
@@ -890,7 +901,8 @@ sub file_name2meta {
   my $composition = $self->file_name2composition(@file_names);
   my $meta = {
     id_iseq_product      => $composition->digest,
-    iseq_composition_tmp => $composition->freeze
+    iseq_composition_tmp => $composition->freeze,
+    composition          => $composition
   };
   if (@file_names == 1) {
     $meta->{'id_run'} = $composition->get_component(0)->id_run;
@@ -899,6 +911,55 @@ sub file_name2meta {
   return $meta;
 }
 
+sub create_component_linking_rows {
+  my ($self, $composition) = @_;
+
+  # Dynamically load classes from npg_tracking::glossary::composition
+  # namespace since this package should not have hard dependency on
+  # other non-CPAN packages.
+  for (grep { not m{factory}smx } @ADDITIONAL_CLASSES) {
+    load_class($_)
+  };
+
+  # The code in this class always suplies the composition
+  # object to this method. This method will also be used to
+  # create missing linking rows, then it's reasonable to
+  # compute the composition object.
+  $composition ||= _json2composition($self->iseq_composition_tmp);
+
+  my $digest         = $self->id_iseq_product;
+  my $num_components = $composition->num_components;
+  my $index          = 1;
+
+  my $lcomponent_rs = $self->result_source->schema
+                           ->resultset('IseqExternalProductComponent');
+
+  foreach my $component ($composition->components_list()) {
+    my $linking_ref = {
+      id_iseq_product_ext => $digest,
+      num_components      => $num_components,
+      component_index     => $index++,
+      id_iseq_product     => ($num_components == 1) ? $digest :
+        npg_tracking::glossary::composition->new(
+          components => [$component])->digest
+    };
+    # There is a chance that the row already exists.
+    # We do not want to error bacause of this, neither we want
+    # to update the record.
+    $lcomponent_rs->find_or_create($linking_ref);
+  }
+
+  return;
+}
+
+sub _json2composition {
+  my $cjson = shift;
+  return npg_tracking::glossary::composition->thaw(
+    $cjson,
+    component_class => 'npg_tracking::glossary::composition::component::illumina'
+  );
+}
+
 __PACKAGE__->meta->make_immutable;
 
 1;
@@ -948,6 +1009,12 @@ a product that is a result of the merge of these file.
 Returns a hash with metadata (run id, composition JSON and
 composition digest) appropriate for this row.
 
+=head2 create_component_linking_rows
+
+For this row, creates missing linking records in the
+iseq_external_product_component table. Takes a a composition
+object corresponding to this row as an optional argument.
+
 =head1 DEPENDENCIES
 
 =over