feat: add basic metrics (#641)

* feat: add basic metrics

* fix: add some custom metrics

package.json

@@ -21,6 +21,7 @@
   "prettier": "@snapshot-labs/prettier-config",
   "dependencies": {
     "@snapshot-labs/keycard": "^0.2.0",
+    "@snapshot-labs/snapshot-metrics": "^1.0.0",
     "@snapshot-labs/snapshot-sentry": "^1.1.0",
     "@snapshot-labs/snapshot.js": "^0.5.3",
     "bluebird": "^3.7.2",

src/helpers/metrics.ts

@@ -0,0 +1,111 @@
+import init, { client } from '@snapshot-labs/snapshot-metrics';
+import { Express } from 'express';
+import { spacesMetadata } from './spaces';
+import { strategies } from './strategies';
+import db from './mysql';
+
+export default function initMetrics(app: Express) {
+  init(app, {
+    normalizedPath: [
+      ['^/api/scores/.+', '/api/scores/#id'],
+      ['^/api/spaces/([^/]+)(/poke)?$', '/api/spaces/#key$2']
+    ],
+    whitelistedPath: [
+      /^\/$/,
+      /^\/api$/,
+      /^\/api\/(msg|message)$/,
+      /^\/api\/spaces\/.+$/,
+      /^\/graphql/
+    ]
+  });
+}
+
+new client.Gauge({
+  name: 'spaces_per_status_count',
+  help: 'Number of spaces per status',
+  labelNames: ['status'],
+  async collect() {
+    const verifiedCount = Object.values(spacesMetadata).filter((s: any) => s.verified).length;
+    this.set({ status: 'verified' }, verifiedCount);
+    this.set({ status: 'unverified' }, Object.keys(spacesMetadata).length - verifiedCount);
+  }
+});
+
+new client.Gauge({
+  name: 'spaces_per_network_count',
+  help: 'Number of spaces per network',
+  labelNames: ['network'],
+  async collect() {
+    const results = {};
+    Object.values(spacesMetadata).forEach((space: any) => {
+      space.networks.forEach(network => {
+        results[network] ||= 0;
+        results[network]++;
+      });
+    });
+
+    for (const r in results) {
+      this.set({ network: r }, results[r]);
+    }
+  }
+});
+
+new client.Gauge({
+  name: 'spaces_per_category_count',
+  help: 'Number of spaces per category',
+  labelNames: ['category'],
+  async collect() {
+    const results = {};
+    Object.values(spacesMetadata).forEach((space: any) => {
+      space.categories.forEach(category => {
+        results[category] ||= 0;
+        results[category]++;
+      });
+    });
+
+    for (const r in results) {
+      this.set({ category: r }, results[r]);
+    }
+  }
+});
+
+new client.Gauge({
+  name: 'proposals_per_status_count',
+  help: 'Number of proposals per status',
+  labelNames: ['status'],
+  async collect() {
+    let results = 0;
+    Object.values(spacesMetadata).forEach((space: any) => {
+      results += space.counts.activeProposals || 0;
+    });
+
+    this.set({ status: 'active' }, results);
+  }
+});
+
+new client.Gauge({
+  name: 'proposals_total_count',
+  help: 'Total number of proposals',
+  async collect() {
+    this.set((await db.queryAsync('SELECT COUNT(id) as count FROM proposals'))[0].count);
+  }
+});
+
+new client.Gauge({
+  name: 'users_total_count',
+  help: 'Total number of users',
+  async collect() {
+    this.set((await db.queryAsync('SELECT COUNT(id) as count FROM users'))[0].count);
+  }
+});
+
+new client.Gauge({
+  name: 'strategies_per_space_count',
+  help: 'Number of strategies per spaces',
+  labelNames: ['name'],
+  async collect() {
+    strategies.forEach((s: any) => {
+      this.set({ name: s.id }, s.spacesCount);
+    });
+  }
+});

src/index.ts

@@ -1,20 +1,23 @@
 import 'dotenv/config';
 import cors from 'cors';
 import express from 'express';
+import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry';
 import api from './api';
 import graphql from './graphql';
 import rateLimit from './helpers/rateLimit';
 import log from './helpers/log';
-import './helpers/strategies';
+import initMetrics from './helpers/metrics';
 import { checkKeycard } from './helpers/keycard';
 import './helpers/moderation';
-import { initLogger, fallbackLogger } from '@snapshot-labs/snapshot-sentry';
+import './helpers/strategies';
 
 const app = express();
 const PORT = process.env.PORT || 3000;
 
 initLogger(app);
+initMetrics(app);
 
+app.disable('x-powered-by');
 app.use(express.json({ limit: '20mb' }));
 app.use(express.urlencoded({ limit: '20mb', extended: false }));
 app.use(cors({ maxAge: 86400 }));

yarn.lock

@@ -1298,6 +1298,14 @@
   resolved "https://registry.yarnpkg.com/@snapshot-labs/prettier-config/-/prettier-config-0.1.0-beta.7.tgz#c8e07e7e9baabee245020a72ac05835b65139823"
   integrity sha512-k/FUf4VWhwLFUmKuWs2mNvmPe691hqhvCJuujD4TfbIivWysmL1TqthwfdQUrQEAQUqVQ2ZKEiGkbufp5J27eQ==
 
+"@snapshot-labs/snapshot-metrics@^1.0.0":
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/@snapshot-labs/snapshot-metrics/-/snapshot-metrics-1.0.0.tgz#1f88a6aacc81f639f7059c153b53c550934cd3b3"
+  integrity sha512-6T8a2NX6Qo6zVAoNIWV8eSJCukCynI/HCLp37VZTzX8jwU/ahGsiDTQC3I/MDus+LDB+8pI5nju33NwM8Q7n2g==
+  dependencies:
+    express-prom-bundle "^6.6.0"
+    prom-client "^14.2.0"
+
 "@snapshot-labs/snapshot-sentry@^1.1.0":
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/@snapshot-labs/snapshot-sentry/-/snapshot-sentry-1.1.0.tgz#b357d4789ffd287f90fb70e7f0b207b47627cf24"
@@ -1834,6 +1842,11 @@ binary-extensions@^2.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
   integrity sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==
 
+[email protected]:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8"
+  integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==
+
 [email protected], bluebird@^3.7.2:
   version "3.7.2"
   resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f"
@@ -2603,6 +2616,14 @@ express-graphql@^0.12.0:
     http-errors "1.8.0"
     raw-body "^2.4.1"
 
+express-prom-bundle@^6.6.0:
+  version "6.6.0"
+  resolved "https://registry.yarnpkg.com/express-prom-bundle/-/express-prom-bundle-6.6.0.tgz#9c33c1bd1478d70e3961a53aed2d17f15ef821ca"
+  integrity sha512-tZh2P2p5a8/yxQ5VbRav011Poa4R0mHqdFwn9Swe/obXDe5F0jY9wtRAfNYnqk4LXY7akyvR/nrvAHxQPWUjsQ==
+  dependencies:
+    on-finished "^2.3.0"
+    url-value-parser "^2.0.0"
+
 express-rate-limit@^5.2.6:
   version "5.2.6"
   resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-5.2.6.tgz#b454e1be8a252081bda58460e0a25bf43ee0f7b0"
@@ -4031,7 +4052,7 @@ object-inspect@^1.9.0:
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea"
   integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==
 
-[email protected]:
+[email protected], on-finished@^2.3.0:
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f"
   integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==
@@ -4229,6 +4250,13 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
   integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
 
+prom-client@^14.2.0:
+  version "14.2.0"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-14.2.0.tgz#ca94504e64156f6506574c25fb1c34df7812cf11"
+  integrity sha512-sF308EhTenb/pDRPakm+WgiN+VdM/T1RaHj1x+MvAuT8UiQP8JmOEbxVqtkbfR4LrvOg5n7ic01kRBDGXjYikA==
+  dependencies:
+    tdigest "^0.1.1"
+
 prompts@^2.0.1:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069"
@@ -4788,6 +4816,13 @@ symbol-observable@^4.0.0:
   resolved "https://registry.yarnpkg.com/symbol-observable/-/symbol-observable-4.0.0.tgz#5b425f192279e87f2f9b937ac8540d1984b39205"
   integrity sha512-b19dMThMV4HVFynSAM1++gBHAbk2Tc/osgLIBZMKsyqh34jb2e8Os7T6ZW/Bt3pJFdBTd2JwAnAAEQV7rSNvcQ==
 
+tdigest@^0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced"
+  integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==
+  dependencies:
+    bintrees "1.0.2"
+
 test-exclude@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e"
@@ -4996,6 +5031,11 @@ uri-js@^4.2.2:
   dependencies:
     punycode "^2.1.0"
 
+url-value-parser@^2.0.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/url-value-parser/-/url-value-parser-2.2.0.tgz#f38ae8cd24604ec69bc219d66929ddbbd93a2b32"
+  integrity sha512-yIQdxJpgkPamPPAPuGdS7Q548rLhny42tg8d4vyTNzFqvOnwqrgHXvgehT09U7fwrzxi3RxCiXjoNUNnNOlQ8A==
+
 util-deprecate@^1.0.1, util-deprecate@~1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"