feat: last running start and stop times for nodes

src/common/node.ts

@@ -73,8 +73,9 @@ type Node = {
    * Timestamp the node was first created, UTC milliseconds
    */
   createdTimestampMs: number;
-  lastStarted?: string;
-  lastStopped?: string;
+  lastRunningTimestampMs?: number;
+  lastStartedTimestampMs?: number;
+  lastStoppedTimestampMs?: number;
   stoppedBy?: NodeStoppedBy;
 };
 type NodeMap = Record<string, Node>;
@@ -99,8 +100,24 @@ export type NodePackage = {
    * Timestamp the node was first created, UTC milliseconds
    */
   createdTimestampMs: number;
-  lastStarted?: string;
-  lastStopped?: string;
+  /**
+   * When the Node Package was most recently detected as running properly.
+   * Definition: "running properly" means the node was running for at least 30 seconds
+   */
+  lastRunningTimestampMs?: number;
+  /**
+   * When the Node Package was most recently started.
+   * (Does not indicate that it successfully started, see lastRunningTimestampMs)
+   */
+  lastStartedTimestampMs?: number;
+  /**
+   * When the Node Package was most recently stopped.
+   */
+  lastStoppedTimestampMs?: number;
+  /**
+   * Sets what stopped the Node Package.
+   * Examples: 'shutdown', 'user', 'crash', or undefined if the node is running
+   */
   stoppedBy?: NodeStoppedBy;
 };
 export type NodePackageMap = Record<string, NodePackage>;

src/main/cronJobs.ts

@@ -14,6 +14,9 @@ type NodeReportData = {
   status: NodeStatus;
   diskUsedGBs?: number;
   network?: string;
+  lastRunningTimestampMs?: number;
+  lastStartedTimestampMs?: number;
+  lastStoppedTimestampMs?: number;
 };
 
 type PackageReportData = {
@@ -22,6 +25,9 @@ type PackageReportData = {
   status: NodeStatus;
   nodes: Record<NodeId, NodeReportData>;
   network?: string;
+  lastRunningTimestampMs?: number;
+  lastStartedTimestampMs?: number;
+  lastStoppedTimestampMs?: number;
 };
 export const reportDataForNodePackages = (
   userNodePackages: UserNodePackages,
@@ -35,6 +41,9 @@ export const reportDataForNodePackages = (
       specVersion: nodePackage.spec.version,
       status: nodePackage.status,
       network: nodePackage.config?.configValuesMap?.network,
+      lastRunningTimestampMs: nodePackage.lastRunningTimestampMs,
+      lastStartedTimestampMs: nodePackage.lastStartedTimestampMs,
+      lastStoppedTimestampMs: nodePackage.lastStoppedTimestampMs,
       nodes: {},
     };
 
@@ -47,7 +56,10 @@ export const reportDataForNodePackages = (
         specId: node.spec.specId,
         specVersion: node.spec.version,
         status: node.status,
-        network: nodePackage.config?.configValuesMap?.network,
+        network: node.config?.configValuesMap?.network,
+        lastRunningTimestampMs: node.lastRunningTimestampMs,
+        lastStartedTimestampMs: node.lastStartedTimestampMs,
+        lastStoppedTimestampMs: node.lastStoppedTimestampMs,
         diskUsedGBs,
       };
     });

src/main/node/setLastRunningTime.ts

@@ -0,0 +1,59 @@
+import Node, { NodeId, NodePackage, NodeStatus } from '../../common/node';
+import logger from '../logger';
+import { registerExitHandler } from '../processExit';
+import { getNodePackage, updateNodePackage } from '../state/nodePackages';
+import { getNode, updateNode } from '../state/nodes';
+
+/**
+ * The time used to determine that a node or nodeService
+ *  is successfully running.
+ * Current value: 30 seconds
+ */
+export const DEFINED_NODE_RUNNING_WAIT_TIME = 30000; // 30 seconds
+
+const timeouts: ReturnType<typeof setTimeout>[] = [];
+
+/**
+ * This wait a specific amount of time and then check to see if a node
+ * or nodeService is still running. If it is still running, then it
+ * will set lastRunningTimestampMs on the node or nodeService.
+ * see DEFINED_NODE_RUNNING_WAIT_TIME for the time used to determine
+ * that it is successfully running.
+ * @param nodeId
+ * @param type
+ */
+export const setLastRunningTime = async (
+  nodeId: NodeId,
+  type: 'node' | 'nodeService',
+) => {
+  logger.info(`setLastRunningTime called for nodeId: ${nodeId}`);
+  timeouts.push(
+    setTimeout(() => {
+      let node: NodePackage | Node | undefined;
+      if (type === 'node') {
+        node = getNodePackage(nodeId);
+      }
+      if (type === 'nodeService') {
+        node = getNode(nodeId);
+      }
+      if (node !== undefined && node.status === NodeStatus.running) {
+        logger.info(`setLastRunningTime, still running nodeId: ${nodeId}`);
+        node.lastRunningTimestampMs = Date.now();
+        if (type === 'node') {
+          updateNodePackage(node as NodePackage);
+        }
+        if (type === 'nodeService') {
+          updateNode(node as Node);
+        }
+      } else {
+        logger.info(`setLastRunningTime, not running(!) nodeId: ${nodeId}`);
+      }
+    }, DEFINED_NODE_RUNNING_WAIT_TIME), // wait 30 seconds
+  );
+};
+
+const onAppClose = () => {
+  timeouts.forEach((timeout) => clearTimeout(timeout));
+};
+
+registerExitHandler(onAppClose);

src/main/nodeManager.ts

@@ -27,6 +27,7 @@ import { ConfigValuesMap, ConfigTranslationMap } from '../common/nodeConfig';
 import { checkNodePortsAndNotify } from './ports';
 import { getNodeLibrary } from './state/nodeLibrary';
 import { getSetPortHasChanged } from './state/nodes';
+import { setLastRunningTime } from './node/setLastRunningTime';
 
 export const addNode = async (
   nodeSpec: NodeSpecification,
@@ -107,6 +108,7 @@ export const startNode = async (nodeId: NodeId) => {
   }
   logger.info(`Starting node ${JSON.stringify(node)}`);
   node.status = NodeStatus.starting;
+  node.lastStartedTimestampMs = Date.now();
   node.stoppedBy = undefined;
   nodeStore.updateNode(node);
 
@@ -117,6 +119,7 @@ export const startNode = async (nodeId: NodeId) => {
       const containerIds = await startPodmanNode(dockerNode);
       dockerNode.runtime.processIds = containerIds;
       dockerNode.status = NodeStatus.running;
+      setLastRunningTime(nodeId, 'nodeService');
       if (getSetPortHasChanged(dockerNode)) {
         checkNodePortsAndNotify(dockerNode);
       }
@@ -126,6 +129,7 @@ export const startNode = async (nodeId: NodeId) => {
     logger.error(err);
     node.status = NodeStatus.errorStarting;
     nodeStore.updateNode(node);
+    throw err;
   }
 };
 
@@ -136,6 +140,7 @@ export const stopNode = async (nodeId: NodeId, stoppedBy: NodeStoppedBy) => {
   }
   logger.info(`Stopping node ${JSON.stringify(node)}`);
   node.status = NodeStatus.stopping;
+  node.lastStoppedTimestampMs = Date.now();
   node.stoppedBy = stoppedBy;
   nodeStore.updateNode(node);
 
@@ -305,9 +310,6 @@ export const initialize = async () => {
             node.status = NodeStatus.stopped;
             // console.log('checkNode: stoppeds', node);
           }
-          if (containerDetails?.State?.FinishedAt) {
-            node.lastStopped = containerDetails?.State?.FinishedAt;
-          }
           compareSpecsAndUpdate(
             node,
             nodeLibrary[node.spec.specId].configTranslation,

src/main/nodePackageManager.ts

@@ -26,6 +26,7 @@ import {
 import { addNode, removeNode, startNode, stopNode } from './nodeManager';
 import { createJwtSecretAtDirs } from './util/jwtSecrets';
 import { ConfigValuesMap } from '../common/nodeConfig';
+import { setLastRunningTime } from './node/setLastRunningTime';
 
 // Created when adding a node and is used to pair a node spec and config
 // for a specific node package service
@@ -140,6 +141,7 @@ export const startNodePackage = async (nodeId: NodeId) => {
   logger.info(`Starting node ${JSON.stringify(node)}`);
   let nodePackageStatus = NodeStatus.starting;
   node.status = nodePackageStatus;
+  node.lastStartedTimestampMs = Date.now();
   node.stoppedBy = undefined;
   nodePackageStore.updateNodePackage(node);
 
@@ -156,6 +158,11 @@ export const startNodePackage = async (nodeId: NodeId) => {
       // throw e;
     }
   }
+  // If all node services start without error, the package is considered running
+  if (nodePackageStatus === NodeStatus.running) {
+    setLastRunningTime(nodeId, 'node');
+  }
+
   // set node status
   node.status = nodePackageStatus;
   nodePackageStore.updateNodePackage(node);
@@ -172,6 +179,7 @@ export const stopNodePackage = async (
   logger.info(`Stopping node ${JSON.stringify(node)}`);
   let nodePackageStatus = NodeStatus.stopping;
   node.status = nodePackageStatus;
+  node.lastStoppedTimestampMs = Date.now();
   node.stoppedBy = stoppedBy;
   nodePackageStore.updateNodePackage(node);