diff --git a/lib/HTFeed/Stage/Collate.pm b/lib/HTFeed/Stage/Collate.pm index a12d8357..81b5a587 100644 --- a/lib/HTFeed/Stage/Collate.pm +++ b/lib/HTFeed/Stage/Collate.pm @@ -50,7 +50,7 @@ sub run{ $self->log_repeat($storage); } $self->_set_done(); - $self->{job_metrics}->inc("ingest_collate_items"); + $self->{job_metrics}->inc("ingest_collate_items_total"); return $self->succeeded(); } diff --git a/lib/HTFeed/Stage/ImageRemediate.pm b/lib/HTFeed/Stage/ImageRemediate.pm index 14388f53..04dd8cc2 100644 --- a/lib/HTFeed/Stage/ImageRemediate.pm +++ b/lib/HTFeed/Stage/ImageRemediate.pm @@ -377,8 +377,8 @@ sub _remediate_tiff { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; my $labels = {format => 'tiff'}; - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", $infile_size, $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels); @@ -438,10 +438,10 @@ sub repair_tiff_exiftool { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; my $labels = {format => 'tiff'}; - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", $infile_size, $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels); - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); return $write_return; } @@ -468,8 +468,8 @@ sub repair_tiff_imagemagick { my $labels = {format => 'tiff', tool => 'imagemagick'}; $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s $infile, $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels); - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); croak("failed repairing $infile\n") if $rval; # Some metadata may be lost when imagemagick compresses infile to outfile. @@ -492,8 +492,8 @@ sub repair_tiff_imagemagick { $labels = {format => 'tiff', tool => 'exiftool'}; $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s $infile, $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels); - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); return !$rval; } @@ -621,10 +621,10 @@ sub _remediate_jpeg2000 { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; my $labels = {format => 'jpeg2000'}; - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", $infile_size, $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels); - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); return $ret_val; } @@ -796,8 +796,8 @@ sub expand_lossless_jpeg2000 { $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s "$path/$jpeg2000", $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s "$path/$tiff", $labels); my $delta_time = $self->{job_metrics}->time - $start_time; - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); # try to compress the TIFF -> JPEG2000 get_logger()->trace("Compressing $path/$tiff to $path/$jpeg2000"); @@ -824,8 +824,8 @@ sub expand_lossless_jpeg2000 { $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s "$path/$tiff", $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s "$path/$jpeg2000_remediated", $labels); $delta_time = $self->{job_metrics}->time - $start_time; - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); $start_time = $self->{job_metrics}->time; # copy all headers from the original jpeg2000 @@ -838,8 +838,8 @@ sub expand_lossless_jpeg2000 { $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s "$path/$tiff", $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s "$path/$jpeg2000_remediated", $labels); $delta_time = $self->{job_metrics}->time - $start_time; - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); # gotta do metrics first or we can't get file sizes rename("$path/$jpeg2000_remediated","$path/$jpeg2000"); @@ -887,10 +887,10 @@ sub expand_other_file_formats { }; my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", $infile_size, $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); } } } @@ -1067,8 +1067,8 @@ sub remediate_tiffs { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; my $labels = {format => "tiff", tool => 'jhove'}; - $self->{job_metrics}->add("ingest_imageremediate_seconds", $delta_time, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_items", $labels); + $self->{job_metrics}->add("ingest_imageremediate_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_imageremediate_items_total", $labels); } sub convert_tiff_to_jpeg2000 { @@ -1139,7 +1139,7 @@ sub convert_tiff_to_jpeg2000 { -s "$infile.unc.tif", $labels ); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); if ($magick_compress_err) { $self->set_error( @@ -1170,7 +1170,7 @@ sub convert_tiff_to_jpeg2000 { $labels = {converted => "tiff->jpeg2000", tool => "grk_compress"}; $self->{job_metrics}->add("ingest_imageremediate_bytes_r_total", -s "$infile.unc.tif", $labels); $self->{job_metrics}->add("ingest_imageremediate_bytes_w_total", -s $outfile, $labels); - $self->{job_metrics}->inc("ingest_imageremediate_images", $labels); + $self->{job_metrics}->inc("ingest_imageremediate_images_total", $labels); # then set new metadata fields - the rest will automatically be # set from the JP2 diff --git a/lib/HTFeed/Stage/Pack.pm b/lib/HTFeed/Stage/Pack.pm index b0986d44..71859bbf 100644 --- a/lib/HTFeed/Stage/Pack.pm +++ b/lib/HTFeed/Stage/Pack.pm @@ -106,11 +106,11 @@ sub zip{ my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; - $self->{job_metrics}->add("ingest_pack_seconds", $delta_time); + $self->{job_metrics}->add("ingest_pack_seconds_total", $delta_time); my $bytes_w = -s $zip_path; $self->{job_metrics}->add("ingest_pack_bytes_r_total", $bytes_r); $self->{job_metrics}->add("ingest_pack_bytes_w_total", $bytes_w); - $self->{job_metrics}->inc("ingest_pack_items"); + $self->{job_metrics}->inc("ingest_pack_items_total"); # success get_logger()->trace("Packing $zip_stage/$pt_objid to $zip_path succeeded"); return 1; diff --git a/lib/HTFeed/Stage/ReMETS.pm b/lib/HTFeed/Stage/ReMETS.pm index 483792a0..a7315cc9 100644 --- a/lib/HTFeed/Stage/ReMETS.pm +++ b/lib/HTFeed/Stage/ReMETS.pm @@ -28,11 +28,13 @@ sub new { } sub run { - my $self = shift; - my $volume = $self->{volume}; + my $self = shift; + + my $volume = $self->{volume}; + my $start_time = $self->{job_metrics}->time; # reset md5 checksum source to repo METS file?? # add uplift METS events; mess with PREMIS events we will include. - my $nspkg = $volume->get_nspkg(); + my $nspkg = $volume->get_nspkg(); my $pkg_class = ref($nspkg); no strict 'refs'; my $pkg_config = ${"${pkg_class}::config"}; @@ -41,7 +43,7 @@ sub run { 'mets_update' ]; - my $old_xpc = $volume->get_repository_mets_xpc(); + my $old_xpc = $volume->get_repository_mets_xpc(); $self->{old_xpc} = $old_xpc; $volume->record_premis_event('mets_update'); @@ -54,9 +56,9 @@ sub run { $mets_stage->run(); # compare new METS to old mets. must have: - my $new_mets_file = $mets_stage->{outfile}; + my $new_mets_file = $mets_stage->{outfile}; $self->{new_mets_file} = $new_mets_file; - my $new_xpc = $volume->_parse_xpc($new_mets_file); + my $new_xpc = $volume->_parse_xpc($new_mets_file); # make sure file count, page count, objid are the same my @queries = ( @@ -78,11 +80,11 @@ sub run { # is this an event we're migrating? my $migrate_events = $nspkg->get('migrate_events'); foreach my $event_type_node ($old_xpc->findnodes("//premis:eventType")) { - my $old_event_type = $event_type_node->textContent(); + my $old_event_type = $event_type_node->textContent(); my @new_event_types = ($old_event_type); if(my $new_event_tags = $migrate_events->{$old_event_type}) { - @new_event_types = map { $nspkg->get_event_configuration($_)->{type} } @$new_event_tags; + @new_event_types = map { $nspkg->get_event_configuration($_)->{type} } @$new_event_tags; } foreach my $new_event_type (@new_event_types) { @@ -201,9 +203,9 @@ sub run { # move old METS aside my $mets_source = $volume->get_mets_path(); - my $mets_dest = $volume->get_repository_mets_path(); + my $mets_dest = $volume->get_repository_mets_path(); my $mets_old; - if($mets_dest =~ /\.mets\.xml$/) { + if ($mets_dest =~ /\.mets\.xml$/) { $mets_old = $mets_dest; $mets_old =~ s/\.mets\.xml$/.pre_uplift.mets.xml/; } else { @@ -233,7 +235,11 @@ sub run { ); $self->_set_done(); - $self->{job_metrics}->add("ingest_remets_bytes", -s $mets_dest); + my $end_time = $self->{job_metrics}->time; + my $delta_time = $end_time - $start_time; + $self->{job_metrics}->add("ingest_remets_seconds_total", $delta_time); + $self->{job_metrics}->add("ingest_remets_bytes_w_total", -s $mets_dest); + $self->{job_metrics}->inc("ingest_remets_items_total"); return $self->succeeded(); } diff --git a/lib/HTFeed/Stage/Unpack.pm b/lib/HTFeed/Stage/Unpack.pm index 1384e20f..837a0a27 100644 --- a/lib/HTFeed/Stage/Unpack.pm +++ b/lib/HTFeed/Stage/Unpack.pm @@ -95,10 +95,10 @@ sub _extract_file { # We don't use $self here, so we have to get job_metrics a different way. my $end_time = $job_metrics->time; my $delta_time = $end_time - $start_time; - $job_metrics->add("ingest_unpack_seconds", $delta_time); + $job_metrics->add("ingest_unpack_seconds_total", $delta_time); my $outdir_size = $job_metrics->dir_size($outdir); $job_metrics->add("ingest_unpack_bytes_w_total", $outdir_size); - $job_metrics->inc("ingest_unpack_items"); + $job_metrics->inc("ingest_unpack_items_total"); get_logger()->trace("Extracting $infile succeeded"); return 1; } diff --git a/lib/HTFeed/Storage.pm b/lib/HTFeed/Storage.pm index 49709f5e..61adadaf 100644 --- a/lib/HTFeed/Storage.pm +++ b/lib/HTFeed/Storage.pm @@ -120,10 +120,10 @@ sub encrypt { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; my $labels = {name => $self->{name}}; - $self->{job_metrics}->add($metric_base . "_seconds", $delta_time, $labels); - $self->{job_metrics}->add($bytes_base . "_read", $bytes_read, $labels); - $self->{job_metrics}->add($bytes_base . "_write", $bytes_write, $labels); - $self->{job_metrics}->inc($metric_base . "_items", $labels); + $self->{job_metrics}->add($metric_base . "_seconds_total", $delta_time, $labels); + $self->{job_metrics}->add($bytes_base . "_r_total", $bytes_read, $labels); + $self->{job_metrics}->add($bytes_base . "_w_total", $bytes_write, $labels); + $self->{job_metrics}->inc($metric_base . "_items_total", $labels); get_logger->debug(" finished encrypt"); return $success; @@ -175,8 +175,8 @@ sub verify_crypt { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; my $labels = {name => $self->{name}}; - $self->{job_metrics}->add("ingest_verify_crypt_seconds", $delta_time, $labels); - $self->{job_metrics}->inc("ingest_verify_crypt_items", $labels); + $self->{job_metrics}->add("ingest_verify_crypt_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_verify_crypt_items_total", $labels); my $bytes_read = (-s $encrypted) + (-s $self->{mets_source}, $labels); $self->{job_metrics}->add("ingest_verify_crypt_bytes_r_total", $bytes_read, $labels); @@ -234,11 +234,11 @@ sub stage { my $delta_time = $end_time - $start_time; my $bytes = (-s $mets_source) + (-s $zip_source); my $labels = {name => $self->{name}}; - $self->{job_metrics}->inc("ingest_stage_items", $labels); + $self->{job_metrics}->inc("ingest_stage_items_total", $labels); # This is a copy operation: add the same bytes to read/write metrics. $self->{job_metrics}->add("ingest_stage_bytes_r_total", $bytes, $labels); $self->{job_metrics}->add("ingest_stage_bytes_w_total", $bytes, $labels); - $self->{job_metrics}->add("ingest_stage_seconds", $delta_time, $labels); + $self->{job_metrics}->add("ingest_stage_seconds_total", $delta_time, $labels); get_logger()->debug(" finished stage"); return 1; @@ -400,8 +400,8 @@ sub postvalidate { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; my $labels = {name => $self->{name}}; - $self->{job_metrics}->inc("ingest_postvalidate_items", $labels); - $self->{job_metrics}->add("ingest_postvalidate_seconds", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_postvalidate_items_total", $labels); + $self->{job_metrics}->add("ingest_postvalidate_seconds_total", $delta_time, $labels); return $valid; } @@ -433,8 +433,8 @@ sub validate_zip_completeness { my $labels = {name => $self->{name}}; $self->{job_metrics}->add($metric_base . "_bytes_r_total", $bytes_read, $labels); # Not counting bytes written, was already counted by Stage::Unpack - $self->{job_metrics}->add($metric_base . "_seconds", $delta_time, $labels); - $self->{job_metrics}->inc($metric_base . "_items", $labels); + $self->{job_metrics}->add($metric_base . "_seconds_total", $delta_time, $labels); + $self->{job_metrics}->inc($metric_base . "_items_total", $labels); return $ok; } @@ -452,8 +452,8 @@ sub prevalidate { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; my $labels = {name => $self->{name}}; - $self->{job_metrics}->inc("ingest_prevalidate_items", $labels); - $self->{job_metrics}->add("ingest_prevalidate_seconds", $delta_time, $labels); + $self->{job_metrics}->inc("ingest_prevalidate_items_total", $labels); + $self->{job_metrics}->add("ingest_prevalidate_seconds_total", $delta_time, $labels); return $valid; } diff --git a/lib/HTFeed/Storage/LocalPairtree.pm b/lib/HTFeed/Storage/LocalPairtree.pm index 86d2f1b3..7308fcda 100644 --- a/lib/HTFeed/Storage/LocalPairtree.pm +++ b/lib/HTFeed/Storage/LocalPairtree.pm @@ -158,8 +158,8 @@ sub record_audit { my $end_time = $self->{job_metrics}->time; my $delta_time = $end_time - $start_time; - $self->{job_metrics}->inc("ingest_record_audit_items"); - $self->{job_metrics}->add("ingest_record_audit_seconds", $delta_time); + $self->{job_metrics}->inc("ingest_record_audit_items_total"); + $self->{job_metrics}->add("ingest_record_audit_seconds_total", $delta_time); return $res; }