From 7da488f14fc3b27f09e70afba57ca0e0e70510b3 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Wed, 3 Jul 2024 16:55:40 -0700 Subject: [PATCH 1/4] tc --- .../metrics/__sql/queued_jobs_by_label.sql | 1 + torchci/rockset/prodVersions.json | 2 +- torchci/scripts/updateQueueTimes.mjs | 29 ++++++++++++++----- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/torchci/rockset/metrics/__sql/queued_jobs_by_label.sql b/torchci/rockset/metrics/__sql/queued_jobs_by_label.sql index 15879ee2f9..7a51f10376 100644 --- a/torchci/rockset/metrics/__sql/queued_jobs_by_label.sql +++ b/torchci/rockset/metrics/__sql/queued_jobs_by_label.sql @@ -48,6 +48,7 @@ SELECT COUNT(*) AS count, MAX(queue_s) AS avg_queue_s, machine_type, + CURRENT_TIMESTAMP() AS _event_time FROM queued_jobs GROUP BY diff --git a/torchci/rockset/prodVersions.json b/torchci/rockset/prodVersions.json index 886e1081e3..7af912a73d 100644 --- a/torchci/rockset/prodVersions.json +++ b/torchci/rockset/prodVersions.json @@ -64,7 +64,7 @@ "master_commit_red_percent_groups": "601949da23f80a28", "master_jobs_red_avg": "7df76d4b0d79e067", "number_of_force_pushes": "7c12c25f00d85d5d", - "queued_jobs_by_label": "9526771e44a48db3", + "queued_jobs_by_label": "acedda4f886e2e32", "queued_jobs": "2a1fce1642bb412d", "reverts": "f5bc84a10c4065a3", "top_reds": "f1a1f5012d419fc2", diff --git a/torchci/scripts/updateQueueTimes.mjs b/torchci/scripts/updateQueueTimes.mjs index 727928e406..f966e30826 100644 --- a/torchci/scripts/updateQueueTimes.mjs +++ b/torchci/scripts/updateQueueTimes.mjs @@ -1,10 +1,23 @@ // We compute queue times by looking at a snapshot of jobs in CI that are // currently queued and seeing how long they've existed. This approach doesn't -// give us historical data, so write our snapshot regularly to another Rockset -// collection so we can get a view of the queue over time. +// give us historical data, so write our snapshot regularly to s3 so we can get +// a view of the queue over time. +import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; import rockset from "@rockset/client"; import { promises as fs } from "fs"; +export function getS3Client() { + return new S3Client({ + region: "us-east-1", + credentials: { + accessKeyId: process.env.OUR_AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.OUR_AWS_SECRET_ACCESS_KEY, + }, + }); +} + +const s3client = getS3Client(); + async function readJSON(path) { const rawData = await fs.readFile(path); return JSON.parse(rawData); @@ -19,8 +32,10 @@ const response = await client.queryLambdas.executeQueryLambda( {} ); -console.log(response); - -await client.documents.addDocuments("metrics", "queue_times_historical", { - data: response.results, -}); +s3client.send( + new PutObjectCommand({ + Bucket: "ossci-raw-job-status", + Key: `queue_times_historical/${response.results[0]._event_time}`, + Body: JSON.stringify(response.results), + }) +); From e6d00a5ceea629af09837096c4cdc6282dc5c2dc Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 9 Jul 2024 08:54:06 -0700 Subject: [PATCH 2/4] update --- torchci/rockset/metrics/__sql/queued_jobs_by_label.sql | 2 +- torchci/rockset/prodVersions.json | 2 +- torchci/scripts/updateQueueTimes.mjs | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/torchci/rockset/metrics/__sql/queued_jobs_by_label.sql b/torchci/rockset/metrics/__sql/queued_jobs_by_label.sql index 7a51f10376..8f5674ac1e 100644 --- a/torchci/rockset/metrics/__sql/queued_jobs_by_label.sql +++ b/torchci/rockset/metrics/__sql/queued_jobs_by_label.sql @@ -48,7 +48,7 @@ SELECT COUNT(*) AS count, MAX(queue_s) AS avg_queue_s, machine_type, - CURRENT_TIMESTAMP() AS _event_time + CURRENT_TIMESTAMP() AS time FROM queued_jobs GROUP BY diff --git a/torchci/rockset/prodVersions.json b/torchci/rockset/prodVersions.json index 7af912a73d..25c763e07f 100644 --- a/torchci/rockset/prodVersions.json +++ b/torchci/rockset/prodVersions.json @@ -64,7 +64,7 @@ "master_commit_red_percent_groups": "601949da23f80a28", "master_jobs_red_avg": "7df76d4b0d79e067", "number_of_force_pushes": "7c12c25f00d85d5d", - "queued_jobs_by_label": "acedda4f886e2e32", + "queued_jobs_by_label": "faa25dfaf336118a", "queued_jobs": "2a1fce1642bb412d", "reverts": "f5bc84a10c4065a3", "top_reds": "f1a1f5012d419fc2", diff --git a/torchci/scripts/updateQueueTimes.mjs b/torchci/scripts/updateQueueTimes.mjs index f966e30826..0444d6a8c3 100644 --- a/torchci/scripts/updateQueueTimes.mjs +++ b/torchci/scripts/updateQueueTimes.mjs @@ -32,10 +32,13 @@ const response = await client.queryLambdas.executeQueryLambda( {} ); +const unixTime = parseInt( + (new Date(response.results[0].time).getTime() / 1000).toFixed(0) +) s3client.send( new PutObjectCommand({ Bucket: "ossci-raw-job-status", - Key: `queue_times_historical/${response.results[0]._event_time}`, + Key: `queue_times_historical/${unixTime}.json`, Body: JSON.stringify(response.results), }) ); From 3f4f50b497198204326e6aea75c9e02766210e66 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 9 Jul 2024 08:56:37 -0700 Subject: [PATCH 3/4] tc --- torchci/scripts/updateQueueTimes.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchci/scripts/updateQueueTimes.mjs b/torchci/scripts/updateQueueTimes.mjs index 0444d6a8c3..f7acc9a1be 100644 --- a/torchci/scripts/updateQueueTimes.mjs +++ b/torchci/scripts/updateQueueTimes.mjs @@ -34,7 +34,7 @@ const response = await client.queryLambdas.executeQueryLambda( const unixTime = parseInt( (new Date(response.results[0].time).getTime() / 1000).toFixed(0) -) +); s3client.send( new PutObjectCommand({ Bucket: "ossci-raw-job-status", From 5a08b9fe3696d55777c2d4241276d30e47ca5e17 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 9 Jul 2024 09:35:29 -0700 Subject: [PATCH 4/4] tc --- torchci/scripts/updateQueueTimes.mjs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/torchci/scripts/updateQueueTimes.mjs b/torchci/scripts/updateQueueTimes.mjs index f7acc9a1be..8804820da8 100644 --- a/torchci/scripts/updateQueueTimes.mjs +++ b/torchci/scripts/updateQueueTimes.mjs @@ -32,13 +32,13 @@ const response = await client.queryLambdas.executeQueryLambda( {} ); -const unixTime = parseInt( - (new Date(response.results[0].time).getTime() / 1000).toFixed(0) -); -s3client.send( - new PutObjectCommand({ - Bucket: "ossci-raw-job-status", - Key: `queue_times_historical/${unixTime}.json`, - Body: JSON.stringify(response.results), - }) -); +for (const r of response.results) { + const unixTime = parseInt((new Date(r.time).getTime() / 1000).toFixed(0)); + s3client.send( + new PutObjectCommand({ + Bucket: "ossci-raw-job-status", + Key: `queue_times_historical/${r.machine_type}/${unixTime}.json`, + Body: JSON.stringify(r), + }) + ); +}