bug fix for metrics

ucd-library · Jan 31, 2024 · c495468 · c495468
1 parent 895e0fa
commit c495468
Show file tree

Hide file tree

Showing 5 changed files with 97 additions and 79 deletions.
diff --git a/services/fin/dbsync/lib/model.js b/services/fin/dbsync/lib/model.js
@@ -55,6 +55,7 @@ class DbSync {
 
   async setupMetrics() {
     if( config.metrics.enabled !== true ) return;
+    if( !metrics.meterProvider ) return;
 
     // setup metrics
     const meter = metrics.meterProvider.getMeter('default');

diff --git a/services/fin/monitoring/lib/fcrepo.js b/services/fin/monitoring/lib/fcrepo.js
@@ -40,6 +40,10 @@ async function fetchMetrics() {
 }
 
 function ensureInstruments() {
+  if( !metrics.meterProvider ) {
+    return;
+  }
+
   const meter = metrics.meterProvider.getMeter('default');
 
   for( let key in data ) {

diff --git a/services/fin/monitoring/lib/fin.js b/services/fin/monitoring/lib/fin.js
@@ -1,76 +1,82 @@
 const {metrics, pg} = require('@ucd-lib/fin-service-utils');
 const {ValueType} = require('@opentelemetry/api');
 
-const meter = metrics.meterProvider.getMeter('default');
-
-const dbsSyncQueueGauge = meter.createObservableGauge('fin.dbsync.queue-length',  {
-  description: 'Number of events in the dbsync queue table',
-  unit: '',
-  valueType: ValueType.INT,
-});
-
-dbsSyncQueueGauge.addCallback(async result => {
-  result.observe(await getDbsyncQueueLength('event'), {table: 'event_queue'});
-  result.observe(await getDbsyncQueueLength('validate'), {table: 'validate_queue'});
-});
-
-
-const dataModelStats = meter.createObservableGauge('fin.dbsync.data-model-stats',  {
-  description: 'Counts of known data model items in the fin database',
-  unit: '',
-  valueType: ValueType.INT,
-});
-
-dataModelStats.addCallback(async result => {
-  let stats = await getDataModelStats();
-  for( let stat of stats ) {
-    result.observe(parseInt(stat.count), {model: stat.model, status: stat.type});
+function init() {
+  if( !metrics.meterProvider ) {
+    return;
   }
+  const meter = metrics.meterProvider.getMeter('default');
+
+  const dbsSyncQueueGauge = meter.createObservableGauge('fin.dbsync.queue-length',  {
+    description: 'Number of events in the dbsync queue table',
+    unit: '',
+    valueType: ValueType.INT,
+  });
+
+  dbsSyncQueueGauge.addCallback(async result => {
+    result.observe(await getDbsyncQueueLength('event'), {table: 'event_queue'});
+    result.observe(await getDbsyncQueueLength('validate'), {table: 'validate_queue'});
+  });
+
+
+  const dataModelStats = meter.createObservableGauge('fin.dbsync.data-model-stats',  {
+    description: 'Counts of known data model items in the fin database',
+    unit: '',
+    valueType: ValueType.INT,
+  });
+
+  dataModelStats.addCallback(async result => {
+    let stats = await getDataModelStats();
+    for( let stat of stats ) {
+      result.observe(parseInt(stat.count), {model: stat.model, status: stat.type});
+    }
+
+    let counts = await getDataModelCounts();
+    for( let count of counts ) {
+      result.observe(parseInt(count.count), {model: count.model, status: 'total'});
+    }
+  });
+
+  const dbsyncStats = meter.createObservableGauge('fin.dbsync.stats',  {
+    description: 'Counts of dbsync event actions',
+    unit: '',
+    valueType: ValueType.INT,
+  });
+
+  dbsyncStats.addCallback(async result => {
+    let stats = await getDbsyncStats();
+    for( let stat of stats ) {
+      result.observe(parseInt(stat.count), {action: stat.action});
+    };
+  });
+
+  const workflowStats = meter.createObservableGauge('fin.workflow.stats',  {
+    description: 'Worflow type and state counts',
+    unit: '',
+    valueType: ValueType.INT,
+  });
+
+  workflowStats.addCallback(async result => {
+    let stats = await getWorkflowStats();
+    for( let stat of stats ) {
+      result.observe(parseInt(stat.count), {name: stat.name, state: stat.state});
+    };
+  });
+
+  const integrationTestTiming = meter.createObservableGauge('fin.health.integration-test',  {
+    description: 'Integration test timings',
+    unit: 'ms',
+    valueType: ValueType.INT,
+  });
+
+  integrationTestTiming.addCallback(async result => {
+    let stats = await getLatestTimings();
+    for( let stat of stats ) {
+      result.observe(parseInt(stat.timing), {action: stat.action, agent: stat.agent});
+    };
+  });  
+}
 
-  let counts = await getDataModelCounts();
-  for( let count of counts ) {
-    result.observe(parseInt(count.count), {model: count.model, status: 'total'});
-  }
-});
-
-const dbsyncStats = meter.createObservableGauge('fin.dbsync.stats',  {
-  description: 'Counts of dbsync event actions',
-  unit: '',
-  valueType: ValueType.INT,
-});
-
-dbsyncStats.addCallback(async result => {
-  let stats = await getDbsyncStats();
-  for( let stat of stats ) {
-    result.observe(parseInt(stat.count), {action: stat.action});
-  };
-});
-
-const workflowStats = meter.createObservableGauge('fin.workflow.stats',  {
-  description: 'Worflow type and state counts',
-  unit: '',
-  valueType: ValueType.INT,
-});
-
-workflowStats.addCallback(async result => {
-  let stats = await getWorkflowStats();
-  for( let stat of stats ) {
-    result.observe(parseInt(stat.count), {name: stat.name, state: stat.state});
-  };
-});
-
-const integrationTestTiming = meter.createObservableGauge('fin.health.integration-test',  {
-  description: 'Integration test timings',
-  unit: 'ms',
-  valueType: ValueType.INT,
-});
-
-integrationTestTiming.addCallback(async result => {
-  let stats = await getLatestTimings();
-  for( let stat of stats ) {
-    result.observe(parseInt(stat.timing), {action: stat.action, agent: stat.agent});
-  };
-});
 
 async function getDbsyncQueueLength(type) {
   let sql = `SELECT * FROM restapi.dbsync_${type}_queue_size`;
@@ -112,4 +118,6 @@ async function getLatestTimings() {
   result = await pg.query(sql, [id]);
 
   return result.rows;
-}
+}
+
+init();
diff --git a/services/fin/monitoring/lib/fs-test.js b/services/fin/monitoring/lib/fs-test.js
@@ -14,6 +14,9 @@ for( let i = 0; i < BUFFER_SIZE; i++ ) {
 }
 
 async function init() {
+  if( !metrics.meterProvider ) {
+    return;
+  }
   const meter = metrics.meterProvider.getMeter('default');
 
   // create read test file

diff --git a/services/fin/node-utils/lib/metrics/index.js b/services/fin/node-utils/lib/metrics/index.js
@@ -13,6 +13,7 @@ const { FsInstrumentation } = require('@opentelemetry/instrumentation-fs');
 const {
   PeriodicExportingMetricReader,
   ConsoleMetricExporter,
+  MeterProvider
 } = require('@opentelemetry/sdk-metrics');
 const { NodeTracerProvider,  ConsoleSpanExporter,
     SimpleSpanProcessor } = require('@opentelemetry/sdk-trace-node');
@@ -48,24 +49,22 @@ function init() {
   } else if( env.FIN_METRICS_EXPORT_STDOUT === 'true' ) {
     traceExporter = new ConsoleSpanExporter();
     metricExporter = new ConsoleMetricExporter();
+    meterProvider = new MeterProvider({
+      resource: new Resource(resourceAttributes())
+    });
   }
 
   let serviceName = env.FIN_SERVICE_NAME || 'unknown';
 
+  if( !metricExporter ) {
+    return;
+  }
 
-  const sdk = new NodeSDK({
-    // spanProcessor: new BatchSpanProcessor(traceExporter, {
-    //   exportTimeoutMillis : 15000,
-    //   maxExportBatchSize : 512*2,
-    //   maxQueueSize : 2048*2
-    // }),
-    // traceExporter: traceExporter,
+  let config = {
     metricReader: new PeriodicExportingMetricReader({
       exportIntervalMillis: 15000,
       exporter: metricExporter,
     }),
-    // this doesn't seem to work :(
-    // instrumentations: [getNodeAutoInstrumentations()],
     instrumentations : [
       // Express instrumentation expects HTTP layer to be instrumented
       // new HttpInstrumentation(),
@@ -75,7 +74,10 @@ function init() {
     ],
     resource: new Resource(resourceAttributes()),
     serviceName : serviceName
-  });
+  }
+
+
+  const sdk = new NodeSDK(config);
 
   sdk.start();