Rockset migration - move upload for queue_times_historical from rockset to s3

torchci/rockset/metrics/__sql/queued_jobs_by_label.sql

@@ -48,6 +48,7 @@ SELECT
   COUNT(*) AS count,
   MAX(queue_s) AS avg_queue_s,
   machine_type,
+  CURRENT_TIMESTAMP() AS time
 FROM
   queued_jobs
 GROUP BY

torchci/rockset/prodVersions.json

@@ -64,7 +64,7 @@
     "master_commit_red_percent_groups": "601949da23f80a28",
     "master_jobs_red_avg": "7df76d4b0d79e067",
     "number_of_force_pushes": "7c12c25f00d85d5d",
-    "queued_jobs_by_label": "9526771e44a48db3",
+    "queued_jobs_by_label": "faa25dfaf336118a",
     "queued_jobs": "2a1fce1642bb412d",
     "reverts": "f5bc84a10c4065a3",
     "top_reds": "f1a1f5012d419fc2",

torchci/scripts/updateQueueTimes.mjs

@@ -1,10 +1,23 @@
 // We compute queue times by looking at a snapshot of jobs in CI that are
 // currently queued and seeing how long they've existed. This approach doesn't
-// give us historical data, so write our snapshot regularly to another Rockset
-// collection so we can get a view of the queue over time.
+// give us historical data, so write our snapshot regularly to s3 so we can get
+// a view of the queue over time.
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
 import rockset from "@rockset/client";
 import { promises as fs } from "fs";
 
+export function getS3Client() {
+  return new S3Client({
+    region: "us-east-1",
+    credentials: {
+      accessKeyId: process.env.OUR_AWS_ACCESS_KEY_ID,
+      secretAccessKey: process.env.OUR_AWS_SECRET_ACCESS_KEY,
+    },
+  });
+}
+
+const s3client = getS3Client();
+
 async function readJSON(path) {
   const rawData = await fs.readFile(path);
   return JSON.parse(rawData);
@@ -19,8 +32,13 @@ const response = await client.queryLambdas.executeQueryLambda(
   {}
 );
 
-console.log(response);
-
-await client.documents.addDocuments("metrics", "queue_times_historical", {
-  data: response.results,
-});
+for (const r of response.results) {
+  const unixTime = parseInt((new Date(r.time).getTime() / 1000).toFixed(0));
+  s3client.send(
+    new PutObjectCommand({
+      Bucket: "ossci-raw-job-status",
+      Key: `queue_times_historical/${r.machine_type}/${unixTime}.json`,
+      Body: JSON.stringify(r),
+    })
+  );
+}