From 911492aa3c5d7bae6240c1304ed8b1b4fadb09e1 Mon Sep 17 00:00:00 2001 From: Leo Ribeiro Date: Sat, 1 Jun 2024 07:33:42 -0400 Subject: [PATCH] init postmetric --- metrics-collector/package.json | 8 +- metrics-collector/pnpm-lock.yaml | 122 ++++++++++++++++++++++ metrics-collector/src/index.ts | 35 ++++++- metrics-collector/src/npm-metrics.ts | 39 ++++++- metrics-collector/src/post-metric.ts | 45 ++++++++ metrics-collector/src/sonatype-metrics.ts | 26 +++-- 6 files changed, 257 insertions(+), 18 deletions(-) create mode 100644 metrics-collector/src/post-metric.ts diff --git a/metrics-collector/package.json b/metrics-collector/package.json index cc434ba..26e90a7 100644 --- a/metrics-collector/package.json +++ b/metrics-collector/package.json @@ -23,6 +23,10 @@ "faker": "^6.6.6", "pg": "^8.11.5", "ts-node": "^10.9.2", - "typescript": "^5.4.5" + "typescript": "^5.4.5", + "yargs": "^17.7.2" + }, + "devDependencies": { + "@types/yargs": "^17.0.32" } -} \ No newline at end of file +} diff --git a/metrics-collector/pnpm-lock.yaml b/metrics-collector/pnpm-lock.yaml index 65d4212..83eb003 100644 --- a/metrics-collector/pnpm-lock.yaml +++ b/metrics-collector/pnpm-lock.yaml @@ -35,6 +35,14 @@ dependencies: typescript: specifier: ^5.4.5 version: 5.4.5 + yargs: + specifier: ^17.7.2 + version: 17.7.2 + +devDependencies: + '@types/yargs': + specifier: ^17.0.32 + version: 17.0.32 packages: @@ -186,6 +194,16 @@ packages: undici-types: 5.26.5 dev: false + /@types/yargs-parser@21.0.3: + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + dev: true + + /@types/yargs@17.0.32: + resolution: {integrity: sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog==} + dependencies: + '@types/yargs-parser': 21.0.3 + dev: true + /accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} engines: {node: '>= 0.6'} @@ -205,6 +223,18 @@ packages: hasBin: true dev: false + /ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + dev: false + + /ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + dependencies: + color-convert: 2.0.1 + dev: false + /arg@4.1.3: resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==} dev: false @@ -253,6 +283,26 @@ packages: set-function-length: 1.2.2 dev: false + /cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + dev: false + + /color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + dependencies: + color-name: 1.1.4 + dev: false + + /color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + dev: false + /content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -330,6 +380,10 @@ packages: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} dev: false + /emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + dev: false + /encodeurl@1.0.2: resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} engines: {node: '>= 0.8'} @@ -347,6 +401,11 @@ packages: engines: {node: '>= 0.4'} dev: false + /escalade@3.1.2: + resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==} + engines: {node: '>=6'} + dev: false + /escape-html@1.0.3: resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} dev: false @@ -428,6 +487,11 @@ packages: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} dev: false + /get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + dev: false + /get-intrinsic@1.2.4: resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==} engines: {node: '>= 0.4'} @@ -495,6 +559,11 @@ packages: engines: {node: '>= 0.10'} dev: false + /is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + dev: false + /make-error@1.3.6: resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==} dev: false @@ -684,6 +753,11 @@ packages: unpipe: 1.0.0 dev: false + /require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + dev: false + /safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} dev: false @@ -761,6 +835,22 @@ packages: engines: {node: '>= 0.8'} dev: false + /string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + dev: false + + /strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + dependencies: + ansi-regex: 5.0.1 + dev: false + /toidentifier@1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} @@ -838,6 +928,15 @@ packages: engines: {node: '>= 0.8'} dev: false + /wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + dev: false + /wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} dev: false @@ -847,6 +946,29 @@ packages: engines: {node: '>=0.4'} dev: false + /y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + dev: false + + /yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + dev: false + + /yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + dependencies: + cliui: 8.0.1 + escalade: 3.1.2 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + dev: false + /yn@3.1.1: resolution: {integrity: sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==} engines: {node: '>=6'} diff --git a/metrics-collector/src/index.ts b/metrics-collector/src/index.ts index 9a83d5d..e69f102 100644 --- a/metrics-collector/src/index.ts +++ b/metrics-collector/src/index.ts @@ -1,17 +1,46 @@ import * as dotenv from "dotenv"; dotenv.config(); +import yargs from "yargs"; +import { hideBin } from 'yargs/helpers'; + import { collectGhMetrics } from "./gh-metrics"; import { collectNpmMetrics } from "./npm-metrics"; import { collectSonatypeMetrics } from "./sonatype-metrics"; const isLocalPersistence = process.env.PERSIST_LOCAL_FILES === "true"; +interface Arguments { + 'collect-gh': boolean; + 'collect-npm': boolean; + 'collect-sonatype': boolean; +} + +const argv = yargs(hideBin(process.argv)) +.options({ + 'collect-gh': { type: 'boolean', description: 'Collect GitHub metrics', default: false }, + 'collect-npm': { type: 'boolean', description: 'Collect npm metrics', default: false }, + 'collect-sonatype': { type: 'boolean', description: 'Collect Sonatype metrics', default: false } +}) +.argv as Arguments; + async function main() { try { - await collectGhMetrics(isLocalPersistence); - await collectNpmMetrics(isLocalPersistence); - await collectSonatypeMetrics(isLocalPersistence); + const collectGh = argv['collect-gh']; + const collectNpm = argv['collect-npm']; + const collectSonatype = argv['collect-sonatype']; + + const noArgs = !collectGh && !collectNpm && !collectSonatype; + + if (collectGh || noArgs) { + await collectGhMetrics(isLocalPersistence); + } + if (collectNpm || noArgs) { + await collectNpmMetrics(isLocalPersistence); + } + if (collectSonatype || noArgs) { + await collectSonatypeMetrics(isLocalPersistence); + } } catch (error) { console.error(error); } diff --git a/metrics-collector/src/npm-metrics.ts b/metrics-collector/src/npm-metrics.ts index d0ef0c7..93e0bcb 100644 --- a/metrics-collector/src/npm-metrics.ts +++ b/metrics-collector/src/npm-metrics.ts @@ -2,6 +2,7 @@ import * as fs from "fs"; import * as path from "path"; import { createObjectCsvWriter } from "csv-writer"; import { readJsonFile, writeJsonFile } from "./utils"; +import { postMetric } from "./post-metric"; // Define the npm packages to collect metrics for const npmPackages = [ @@ -18,7 +19,10 @@ const npmPackages = [ const dataFilePath = path.join(process.cwd(), "npm_metrics.json"); const csvFilePath = path.join(process.cwd(), "npm_metrics.csv"); -async function collectNpmMetrics(isLocalPersistence: boolean = false) { +async function collectNpmMetrics( + isLocalPersistence: boolean = false, + metricDate?: string +) { const timestamp = new Date().toISOString(); const metrics = []; @@ -28,12 +32,18 @@ async function collectNpmMetrics(isLocalPersistence: boolean = false) { getNpmDownloadCount(pkg, true), getNpmDownloadCount(pkg), ]); + let metricDateDownloads; + if (metricDate) { + metricDateDownloads = await getNpmDownloadCount(pkg, false, metricDate); + } metrics.push({ pkg, timestamp, publishedAt: totalDownloads.start, totalDownloads: totalDownloads.downloads, lastMonthDownloads: lastMonthDownloads.downloads, + metricDate, + metricDateDownloads: metricDateDownloads?.downloads, }); } @@ -52,19 +62,44 @@ async function collectNpmMetrics(isLocalPersistence: boolean = false) { console.log( "NPM metrics have been successfully saved to npm_metrics.json and npm_metrics.csv" ); + } else { + await postNpmMetrics(metrics); } return metrics; } +async function postNpmMetrics(metrics: any) { + for (const metric of metrics) { + const labels = { + package: metric.pkg, + }; + + if (metric.metricDate) { + const metricDate = new Date(metric.metricDate); + await postMetric( + "npm_downloads", + metric.metricDateDownloads || 0, + labels, + metricDate + ); + } + + await postMetric("npm_total_downloads", metric.totalDownloads, labels); + } +} + async function getNpmDownloadCount( packageName: string, - onlyLastMonth?: boolean + onlyLastMonth?: boolean, + singleDay?: string ): Promise<{ downloads: number; start: string }> { try { let url = `https://api.npmjs.org/downloads/point`; if (onlyLastMonth) { url += `/last-month/${packageName}`; + } else if (singleDay) { + url += `/${singleDay}/${packageName}`; } else { url += `/1970-01-01:2100-01-01/${packageName}`; } diff --git a/metrics-collector/src/post-metric.ts b/metrics-collector/src/post-metric.ts new file mode 100644 index 0000000..3344ce8 --- /dev/null +++ b/metrics-collector/src/post-metric.ts @@ -0,0 +1,45 @@ +const metricsServiceAppUrl = 'http://localhost:3001/api/v1' // TODO: change to env + +interface Labels { + [key: string]: string; +} + +interface MetricPayload { + metricName: string; + value: number; + labels: Labels; + timestamp?: string; +} + +export const postMetric = async ( + metricName: string, + value: number, + labels: Labels, + timestamp?: Date +): Promise => { + const payload: MetricPayload = { + metricName, + value: value, + labels: labels, + timestamp: (timestamp || new Date()).toISOString(), + }; + + try { + const response = await fetch(`${metricsServiceAppUrl}/metrics`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + }); + + if (!response.ok) { + throw new Error(`Error posting metric: ${response.statusText}`); + } + + const data = await response.json(); + console.log("Metric posted successfully:", JSON.stringify(payload)); + } catch (error) { + console.error("Error posting metric:", error); + } +}; diff --git a/metrics-collector/src/sonatype-metrics.ts b/metrics-collector/src/sonatype-metrics.ts index efc6b64..cf9ac9f 100644 --- a/metrics-collector/src/sonatype-metrics.ts +++ b/metrics-collector/src/sonatype-metrics.ts @@ -8,24 +8,16 @@ const groupId = "xyz.block"; const dataFilePath = path.join(process.cwd(), "sonatype_metrics.json"); const csvFilePath = path.join(process.cwd(), "sonatype_metrics.csv"); -const sonatypeUsername = process.env.SONATYPE_USERNAME; -const sonatypePassword = process.env.SONATYPE_PASSWORD; - -if (!sonatypeUsername || !sonatypePassword) { - throw new Error( - "SONATYPE_USERNAME and SONATYPE_PASSWORD must be set in environment variables." - ); -} - -const requestHeaders: HeadersInit = { +const requestHeaders: Record = { Accept: "application/json", - Authorization: "Basic " + btoa(`${sonatypeUsername}:${sonatypePassword}`), }; const sonatypeCentralStatsUrl = "https://s01.oss.sonatype.org/service/local/stats"; async function collectSonatypeMetrics(isLocalPersistence: boolean = false) { + initAuth(); + const timestamp = new Date().toISOString(); const projectId = await getProjectId(groupId); const artifacts = await getArtifacts(projectId, groupId); @@ -65,6 +57,18 @@ async function collectSonatypeMetrics(isLocalPersistence: boolean = false) { return metrics; } +const initAuth = () => { + const sonatypeUsername = process.env.SONATYPE_USERNAME; + const sonatypePassword = process.env.SONATYPE_PASSWORD; + if (!sonatypeUsername || !sonatypePassword) { + throw new Error( + "SONATYPE_USERNAME and SONATYPE_PASSWORD must be set in environment variables." + ); + } + requestHeaders.Authorization = + "Basic " + btoa(`${sonatypeUsername}:${sonatypePassword}`); +}; + async function getProjectId(groupId: string): Promise { try { const response = await fetch(`${sonatypeCentralStatsUrl}/projects`, {