update: added verification status to verify job

CHANGELOG.md

@@ -81,6 +81,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Fixed
 
+- refactor: instrumentations
 - `is_worker_enabled` status check moved from `VerificationFailed` to `Failed`
 - refactor: static attributes for telemetry
 - refactor: aws setup for Event Bridge

crates/orchestrator/src/jobs/mod.rs

@@ -186,7 +186,7 @@ pub async fn create_job(
     tracing::info!(log_type = "completed", category = "general", function_type = "create_job", block_no = %internal_id, "General create job completed for block");
     let duration = start.elapsed();
     ORCHESTRATOR_METRICS.block_gauge.record(parse_string(&internal_id)?, &attributes);
-    ORCHESTRATOR_METRICS.successful_jobs.add(1.0, &attributes);
+    ORCHESTRATOR_METRICS.successful_job_operations.add(1.0, &attributes);
     ORCHESTRATOR_METRICS.jobs_response_time.record(duration.as_secs_f64(), &attributes);
     Ok(())
 }
@@ -201,7 +201,7 @@ pub async fn process_job(id: Uuid, config: Arc<Config>) -> Result<(), JobError>
     tracing::info!(log_type = "starting", category = "general", function_type = "process_job", block_no = %internal_id, "General process job started for block");
 
     tracing::Span::current().record("job", format!("{:?}", job.clone()));
-    tracing::Span::current().record("job_type", format!("{:?}", job.job_type.clone()));
+    tracing::Span::current().record("job_type", format!("{:?}", job.job_type));
     tracing::Span::current().record("internal_id", job.internal_id.clone());
 
     tracing::debug!(job_id = ?id, status = ?job.status, "Current job status");
@@ -300,7 +300,7 @@ pub async fn process_job(id: Uuid, config: Arc<Config>) -> Result<(), JobError>
 
     tracing::info!(log_type = "completed", category = "general", function_type = "process_job", block_no = %internal_id, "General process job completed for block");
     let duration = start.elapsed();
-    ORCHESTRATOR_METRICS.successful_jobs.add(1.0, &attributes);
+    ORCHESTRATOR_METRICS.successful_job_operations.add(1.0, &attributes);
     ORCHESTRATOR_METRICS.block_gauge.record(parse_string(&job.internal_id)?, &attributes);
     ORCHESTRATOR_METRICS.jobs_response_time.record(duration.as_secs_f64(), &attributes);
     Ok(())
@@ -341,7 +341,13 @@ pub async fn verify_job(id: Uuid, config: Arc<Config>) -> Result<(), JobError> {
     let job_handler = factory::get_job_handler(&job.job_type).await;
     tracing::debug!(job_id = ?id, "Verifying job with handler");
     let verification_status = job_handler.verify_job(config.clone(), &mut job).await?;
-    tracing::Span::current().record("verification_status", format!("{:?}", verification_status.clone()));
+    tracing::Span::current().record("verification_status", format!("{:?}", &verification_status));
+
+    let attributes = [
+        KeyValue::new("operation_job_type", format!("{:?}", job.job_type)),
+        KeyValue::new("operation_type", "verify_job"),
+        KeyValue::new("operation_verification_status", format!("{:?}", &verification_status)),
+    ];
 
     match verification_status {
         JobVerificationStatus::Verified => {
@@ -437,14 +443,9 @@ pub async fn verify_job(id: Uuid, config: Arc<Config>) -> Result<(), JobError> {
         }
     };
 
-    let attributes = [
-        KeyValue::new("operation_job_type", format!("{:?}", job.job_type)),
-        KeyValue::new("operation_type", "verify_job"),
-    ];
-
     tracing::info!(log_type = "completed", category = "general", function_type = "verify_job", block_no = %internal_id, "General verify job completed for block");
     let duration = start.elapsed();
-    ORCHESTRATOR_METRICS.successful_jobs.add(1.0, &attributes);
+    ORCHESTRATOR_METRICS.successful_job_operations.add(1.0, &attributes);
     ORCHESTRATOR_METRICS.jobs_response_time.record(duration.as_secs_f64(), &attributes);
     ORCHESTRATOR_METRICS.block_gauge.record(parse_string(&job.internal_id)?, &attributes);
     Ok(())
@@ -491,6 +492,9 @@ async fn move_job_to_failed(job: &JobItem, config: Arc<Config>, reason: String)
     {
         Ok(_) => {
             tracing::info!(log_type = "completed", category = "general", function_type = "handle_job_failure", block_no = %internal_id, "General handle job failure completed for block");
+            ORCHESTRATOR_METRICS
+                .failed_jobs
+                .add(1.0, &[KeyValue::new("operation_job_type", format!("{:?}", job.job_type))]);
             Ok(())
         }
         Err(e) => {

crates/orchestrator/src/metrics.rs

@@ -9,7 +9,8 @@ register_metric!(ORCHESTRATOR_METRICS, OrchestratorMetrics);
 
 pub struct OrchestratorMetrics {
     pub block_gauge: Gauge<f64>,
-    pub successful_jobs: Counter<f64>,
+    pub successful_job_operations: Counter<f64>,
+    pub failed_job_operations: Counter<f64>,
     pub failed_jobs: Counter<f64>,
     pub jobs_response_time: Gauge<f64>,
     pub db_calls_response_time: Gauge<f64>,
@@ -35,10 +36,17 @@ impl Metrics for OrchestratorMetrics {
             "block".to_string(),
         );
 
-        let successful_jobs = register_counter_metric_instrument(
+        let successful_job_operations = register_counter_metric_instrument(
             &orchestrator_meter,
-            "successful_jobs".to_string(),
-            "A counter to show count of successful jobs over time".to_string(),
+            "successful_job_operations".to_string(),
+            "A counter to show count of successful job operations over time".to_string(),
+            "jobs".to_string(),
+        );
+
+        let failed_job_operations = register_counter_metric_instrument(
+            &orchestrator_meter,
+            "failed_job_operations".to_string(),
+            "A counter to show count of failed job operations over time".to_string(),
             "jobs".to_string(),
         );
 
@@ -63,6 +71,13 @@ impl Metrics for OrchestratorMetrics {
             "Time".to_string(),
         );
 
-        Self { block_gauge, successful_jobs, failed_jobs, jobs_response_time, db_calls_response_time }
+        Self {
+            block_gauge,
+            successful_job_operations,
+            failed_job_operations,
+            failed_jobs,
+            jobs_response_time,
+            db_calls_response_time,
+        }
     }
 }

crates/orchestrator/src/routes/job_routes.rs

@@ -43,8 +43,7 @@ async fn handle_process_job_request(
             ApiResponse::success(response).into_response()
         }
         Err(e) => {
-            let attributes = [KeyValue::new("operation_type", "process_job")];
-            ORCHESTRATOR_METRICS.failed_jobs.add(1.0, &attributes);
+            ORCHESTRATOR_METRICS.failed_job_operations.add(1.0, &[KeyValue::new("operation_type", "process_job")]);
             ApiResponse::<JobApiResponse>::error(e.to_string()).into_response()
         }
     }
@@ -69,8 +68,7 @@ async fn handle_verify_job_request(
             ApiResponse::success(response).into_response()
         }
         Err(e) => {
-            let attributes = [KeyValue::new("operation_type", "verify_job")];
-            ORCHESTRATOR_METRICS.failed_jobs.add(1.0, &attributes);
+            ORCHESTRATOR_METRICS.failed_job_operations.add(1.0, &[KeyValue::new("operation_type", "verify_job")]);
             ApiResponse::<JobApiResponse>::error(e.to_string()).into_response()
         }
     }

crates/orchestrator/src/workers/data_submission_worker.rs

@@ -34,7 +34,7 @@ impl Worker for DataSubmissionWorker {
                         KeyValue::new("operation_job_type", format!("{:?}", JobType::DataSubmission)),
                         KeyValue::new("operation_type", format!("{:?}", "create_job")),
                     ];
-                    ORCHESTRATOR_METRICS.failed_jobs.add(1.0, &attributes);
+                    ORCHESTRATOR_METRICS.failed_job_operations.add(1.0, &attributes);
                 }
             }
         }

crates/orchestrator/src/workers/proving.rs

@@ -35,7 +35,7 @@ impl Worker for ProvingWorker {
                         KeyValue::new("operation_job_type", format!("{:?}", JobType::ProofCreation)),
                         KeyValue::new("operation_type", format!("{:?}", "create_job")),
                     ];
-                    ORCHESTRATOR_METRICS.failed_jobs.add(1.0, &attributes);
+                    ORCHESTRATOR_METRICS.failed_job_operations.add(1.0, &attributes);
                 }
             }
         }

crates/orchestrator/src/workers/snos.rs

@@ -56,7 +56,7 @@ impl Worker for SnosWorker {
                         KeyValue::new("operation_job_type", format!("{:?}", JobType::SnosRun)),
                         KeyValue::new("operation_type", format!("{:?}", "create_job")),
                     ];
-                    ORCHESTRATOR_METRICS.failed_jobs.add(1.0, &attributes);
+                    ORCHESTRATOR_METRICS.failed_job_operations.add(1.0, &attributes);
                 }
             }
         }

crates/orchestrator/src/workers/update_state.rs

@@ -125,7 +125,7 @@ impl Worker for UpdateStateWorker {
                     KeyValue::new("operation_job_type", format!("{:?}", JobType::StateTransition)),
                     KeyValue::new("operation_type", format!("{:?}", "create_job")),
                 ];
-                ORCHESTRATOR_METRICS.failed_jobs.add(1.0, &attributes);
+                ORCHESTRATOR_METRICS.failed_job_operations.add(1.0, &attributes);
                 return Err(e.into());
             }
         }