implementing new naming convention

hathitrust · Jul 18, 2024 · c7a83f1 · c7a83f1
1 parent 20e42df
commit c7a83f1
Show file tree

Hide file tree

Showing 7 changed files with 60 additions and 54 deletions.
diff --git a/lib/HTFeed/Stage/Collate.pm b/lib/HTFeed/Stage/Collate.pm
@@ -50,7 +50,7 @@ sub run{
 	$self->log_repeat($storage);
     }
     $self->_set_done();
-    $self->{job_metrics}->inc("ingest_collate_items");
+    $self->{job_metrics}->inc("ingest_collate_items_total");
 
     return $self->succeeded();
 }

diff --git a/lib/HTFeed/Stage/ImageRemediate.pm b/lib/HTFeed/Stage/ImageRemediate.pm
@@ -377,8 +377,8 @@ sub _remediate_tiff {
     my $end_time   = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
     my $labels = {format => 'tiff'};
-    $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
-    $self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+    $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
+    $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", $infile_size, $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels);
 
@@ -438,10 +438,10 @@ sub repair_tiff_exiftool {
     my $end_time = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
     my $labels = {format => 'tiff'};
-    $self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+    $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", $infile_size, $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels);
-    $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
+    $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
 
     return $write_return;
 }
@@ -468,8 +468,8 @@ sub repair_tiff_imagemagick {
     my $labels = {format => 'tiff', tool => 'imagemagick'};
     $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s $infile, $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels);
-    $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
-    $self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+    $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
+    $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
     croak("failed repairing $infile\n") if $rval;
 
     # Some metadata may be lost when imagemagick compresses infile to outfile.
@@ -492,8 +492,8 @@ sub repair_tiff_imagemagick {
     $labels = {format => 'tiff', tool => 'exiftool'};
     $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s $infile, $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels);
-    $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
-    $self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+    $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
+    $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
 
     return !$rval;
 }
@@ -621,10 +621,10 @@ sub _remediate_jpeg2000 {
     my $end_time = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
     my $labels = {format => 'jpeg2000'};
-    $self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+    $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", $infile_size, $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels);
-    $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
+    $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
 
     return $ret_val;
 }
@@ -796,8 +796,8 @@ sub expand_lossless_jpeg2000 {
 		$self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s "$path/$jpeg2000", $labels);
                 $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s "$path/$tiff", $labels);
                 my $delta_time = $self->{job_metrics}->time - $start_time;
-                $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
-		$self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+                $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
+		$self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
 
                 # try to compress the TIFF -> JPEG2000
                 get_logger()->trace("Compressing $path/$tiff to $path/$jpeg2000");
@@ -824,8 +824,8 @@ sub expand_lossless_jpeg2000 {
 		$self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s "$path/$tiff", $labels);
                 $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s "$path/$jpeg2000_remediated", $labels);
                 $delta_time = $self->{job_metrics}->time - $start_time;
-                $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
-		$self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+                $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
+		$self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
 
                 $start_time = $self->{job_metrics}->time;
                 # copy all headers from the original jpeg2000
@@ -838,8 +838,8 @@ sub expand_lossless_jpeg2000 {
 		$self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s "$path/$tiff", $labels);
                 $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s "$path/$jpeg2000_remediated", $labels);
                 $delta_time =  $self->{job_metrics}->time - $start_time;
-                $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
-		$self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+                $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
+		$self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
 
                 # gotta do metrics first or we can't get file sizes
                 rename("$path/$jpeg2000_remediated","$path/$jpeg2000");
@@ -887,10 +887,10 @@ sub expand_other_file_formats {
             };
             my $end_time = $self->{job_metrics}->time;
             my $delta_time = $end_time - $start_time;
-            $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
+            $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
 	    $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", $infile_size, $labels);
 	    $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels);
-	    $self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+	    $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
 	}
     }
 }
@@ -1067,8 +1067,8 @@ sub remediate_tiffs {
     my $end_time = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
     my $labels = {format => "tiff", tool => 'jhove'};
-    $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels);
-    $self->{job_metrics}->inc("ingest_imageremediate_items", $labels);
+    $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels);
+    $self->{job_metrics}->inc("ingest_imageremediate_items_total", $labels);
 }
 
 sub convert_tiff_to_jpeg2000 {
@@ -1139,7 +1139,7 @@ sub convert_tiff_to_jpeg2000 {
         -s "$infile.unc.tif",
         $labels
     );
-    $self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+    $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
 
     if ($magick_compress_err) {
 	$self->set_error(
@@ -1170,7 +1170,7 @@ sub convert_tiff_to_jpeg2000 {
     $labels = {converted => "tiff->jpeg2000", tool => "grk_compress"};
     $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s "$infile.unc.tif", $labels);
     $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels);
-    $self->{job_metrics}->inc("ingest_imageremediate_images", $labels);
+    $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels);
 
     # then set new metadata fields - the rest will automatically be
     # set from the JP2

diff --git a/lib/HTFeed/Stage/Pack.pm b/lib/HTFeed/Stage/Pack.pm
@@ -106,11 +106,11 @@ sub zip{
 
     my $end_time = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
-    $self->{job_metrics}->add("ingest_pack_seconds", $delta_time);
+    $self->{job_metrics}->add("ingest_pack_seconds_total", $delta_time);
     my $bytes_w = -s $zip_path;
     $self->{job_metrics}->add("ingest_pack_bytes_r_total", $bytes_r);
     $self->{job_metrics}->add("ingest_pack_bytes_w_total", $bytes_w);
-    $self->{job_metrics}->inc("ingest_pack_items");
+    $self->{job_metrics}->inc("ingest_pack_items_total");
     # success
     get_logger()->trace("Packing $zip_stage/$pt_objid to $zip_path succeeded");
     return 1;

diff --git a/lib/HTFeed/Stage/ReMETS.pm b/lib/HTFeed/Stage/ReMETS.pm
@@ -28,11 +28,13 @@ sub new {
 }
 
 sub run {
-    my $self = shift;
-    my $volume = $self->{volume};
+    my $self   = shift;
+
+    my $volume     = $self->{volume};
+    my $start_time = $self->{job_metrics}->time;
     # reset md5 checksum source to repo METS file??
     # add uplift METS events; mess with PREMIS events we will include.
-    my $nspkg = $volume->get_nspkg();
+    my $nspkg     = $volume->get_nspkg();
     my $pkg_class = ref($nspkg);
     no strict 'refs';
     my $pkg_config = ${"${pkg_class}::config"};
@@ -41,7 +43,7 @@ sub run {
         'mets_update'
     ];
 
-    my $old_xpc = $volume->get_repository_mets_xpc();
+    my $old_xpc      = $volume->get_repository_mets_xpc();
     $self->{old_xpc} = $old_xpc;
 
     $volume->record_premis_event('mets_update');
@@ -54,9 +56,9 @@ sub run {
     $mets_stage->run();
 
     # compare new METS to old mets. must have:
-    my $new_mets_file = $mets_stage->{outfile};
+    my $new_mets_file      = $mets_stage->{outfile};
     $self->{new_mets_file} = $new_mets_file;
-    my $new_xpc = $volume->_parse_xpc($new_mets_file);
+    my $new_xpc            = $volume->_parse_xpc($new_mets_file);
 
     # make sure file count, page count, objid are the same
     my @queries = (
@@ -78,11 +80,11 @@ sub run {
     # is this an event we're migrating?
     my $migrate_events = $nspkg->get('migrate_events');
     foreach my $event_type_node ($old_xpc->findnodes("//premis:eventType")) {
-        my $old_event_type = $event_type_node->textContent();
+        my $old_event_type  = $event_type_node->textContent();
         my @new_event_types = ($old_event_type);
 
         if(my $new_event_tags = $migrate_events->{$old_event_type}) {
-            @new_event_types = map { $nspkg->get_event_configuration($_)->{type} } @$new_event_tags;
+            @new_event_types  = map { $nspkg->get_event_configuration($_)->{type} } @$new_event_tags;
         }
 
         foreach my $new_event_type (@new_event_types) {
@@ -201,9 +203,9 @@ sub run {
 
     # move old METS aside
     my $mets_source = $volume->get_mets_path();
-    my $mets_dest = $volume->get_repository_mets_path();
+    my $mets_dest   = $volume->get_repository_mets_path();
     my $mets_old;
-    if($mets_dest =~ /\.mets\.xml$/) {
+    if ($mets_dest =~ /\.mets\.xml$/) {
         $mets_old = $mets_dest;
         $mets_old =~ s/\.mets\.xml$/.pre_uplift.mets.xml/;
     } else {
@@ -233,7 +235,11 @@ sub run {
 	);
 
         $self->_set_done();
-	$self->{job_metrics}->add("ingest_remets_bytes", -s $mets_dest);
+        my $end_time   = $self->{job_metrics}->time;
+        my $delta_time = $end_time - $start_time;
+        $self->{job_metrics}->add("ingest_remets_seconds_total", $delta_time);
+	$self->{job_metrics}->add("ingest_remets_bytes_w_total", -s $mets_dest);
+        $self->{job_metrics}->inc("ingest_remets_items_total");
 
 	return $self->succeeded();
     }

diff --git a/lib/HTFeed/Stage/Unpack.pm b/lib/HTFeed/Stage/Unpack.pm
@@ -95,10 +95,10 @@ sub _extract_file {
     # We don't use $self here, so we have to get job_metrics a different way.
     my $end_time    = $job_metrics->time;
     my $delta_time  = $end_time - $start_time;
-    $job_metrics->add("ingest_unpack_seconds", $delta_time);
+    $job_metrics->add("ingest_unpack_seconds_total", $delta_time);
     my $outdir_size = $job_metrics->dir_size($outdir);
     $job_metrics->add("ingest_unpack_bytes_w_total", $outdir_size);
-    $job_metrics->inc("ingest_unpack_items");
+    $job_metrics->inc("ingest_unpack_items_total");
     get_logger()->trace("Extracting $infile succeeded");
     return 1;
 }

diff --git a/lib/HTFeed/Storage.pm b/lib/HTFeed/Storage.pm
@@ -120,10 +120,10 @@ sub encrypt {
     my $end_time = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
     my $labels = {name => $self->{name}};
-    $self->{job_metrics}->add($metric_base . "_seconds", $delta_time, $labels);
-    $self->{job_metrics}->add($bytes_base . "_read", $bytes_read, $labels);
-    $self->{job_metrics}->add($bytes_base . "_write", $bytes_write, $labels);
-    $self->{job_metrics}->inc($metric_base . "_items", $labels);
+    $self->{job_metrics}->add($metric_base . "_seconds_total", $delta_time, $labels);
+    $self->{job_metrics}->add($bytes_base . "_r_total", $bytes_read, $labels);
+    $self->{job_metrics}->add($bytes_base . "_w_total", $bytes_write, $labels);
+    $self->{job_metrics}->inc($metric_base . "_items_total", $labels);
     get_logger->debug("  finished encrypt");
 
     return $success;
@@ -175,8 +175,8 @@ sub verify_crypt {
     my $end_time    = $self->{job_metrics}->time;
     my $delta_time  = $end_time - $start_time;
     my $labels = {name => $self->{name}};
-    $self->{job_metrics}->add("ingest_verify_crypt_seconds", $delta_time, $labels);
-    $self->{job_metrics}->inc("ingest_verify_crypt_items", $labels);
+    $self->{job_metrics}->add("ingest_verify_crypt_seconds_total", $delta_time, $labels);
+    $self->{job_metrics}->inc("ingest_verify_crypt_items_total", $labels);
     my $bytes_read = (-s $encrypted) + (-s $self->{mets_source}, $labels);
     $self->{job_metrics}->add("ingest_verify_crypt_bytes_r_total", $bytes_read, $labels);
 
@@ -234,11 +234,11 @@ sub stage {
         my $delta_time = $end_time - $start_time;
         my $bytes      = (-s $mets_source) + (-s $zip_source);
         my $labels     = {name => $self->{name}};
-        $self->{job_metrics}->inc("ingest_stage_items", $labels);
+        $self->{job_metrics}->inc("ingest_stage_items_total", $labels);
         # This is a copy operation: add the same bytes to read/write metrics.
         $self->{job_metrics}->add("ingest_stage_bytes_r_total", $bytes, $labels);
         $self->{job_metrics}->add("ingest_stage_bytes_w_total", $bytes, $labels);
-        $self->{job_metrics}->add("ingest_stage_seconds", $delta_time, $labels);
+        $self->{job_metrics}->add("ingest_stage_seconds_total", $delta_time, $labels);
 
         get_logger()->debug("  finished stage");
         return 1;
@@ -400,8 +400,8 @@ sub postvalidate {
     my $end_time   = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
     my $labels = {name => $self->{name}};
-    $self->{job_metrics}->inc("ingest_postvalidate_items", $labels);
-    $self->{job_metrics}->add("ingest_postvalidate_seconds", $delta_time, $labels);
+    $self->{job_metrics}->inc("ingest_postvalidate_items_total", $labels);
+    $self->{job_metrics}->add("ingest_postvalidate_seconds_total", $delta_time, $labels);
 
     return $valid;
 }
@@ -433,8 +433,8 @@ sub validate_zip_completeness {
     my $labels     = {name => $self->{name}};
     $self->{job_metrics}->add($metric_base . "_bytes_r_total", $bytes_read, $labels);
     # Not counting bytes written, was already counted by Stage::Unpack
-    $self->{job_metrics}->add($metric_base . "_seconds", $delta_time, $labels);
-    $self->{job_metrics}->inc($metric_base . "_items", $labels);
+    $self->{job_metrics}->add($metric_base . "_seconds_total", $delta_time, $labels);
+    $self->{job_metrics}->inc($metric_base . "_items_total", $labels);
 
     return $ok;
 }
@@ -452,8 +452,8 @@ sub prevalidate {
     my $end_time   = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
     my $labels = {name => $self->{name}};
-    $self->{job_metrics}->inc("ingest_prevalidate_items", $labels);
-    $self->{job_metrics}->add("ingest_prevalidate_seconds", $delta_time, $labels);
+    $self->{job_metrics}->inc("ingest_prevalidate_items_total", $labels);
+    $self->{job_metrics}->add("ingest_prevalidate_seconds_total", $delta_time, $labels);
 
     return $valid;
 }

diff --git a/lib/HTFeed/Storage/LocalPairtree.pm b/lib/HTFeed/Storage/LocalPairtree.pm
@@ -158,8 +158,8 @@ sub record_audit {
 
     my $end_time   = $self->{job_metrics}->time;
     my $delta_time = $end_time - $start_time;
-    $self->{job_metrics}->inc("ingest_record_audit_items");
-    $self->{job_metrics}->add("ingest_record_audit_seconds", $delta_time);
+    $self->{job_metrics}->inc("ingest_record_audit_items_total");
+    $self->{job_metrics}->add("ingest_record_audit_seconds_total", $delta_time);
 
     return $res;
 }