Rockset migration - update queue times- explicitly use the new time f…

…ield instead of _event_time and fix env variables in update queue times workflow (#5412) Related to #5398 Change historical queue times query for HUD metrics page Looks good <img width="847" alt="image" src="https://github.com/pytorch/test-infra/assets/44682903/3376331c-01f6-45e9-a22c-da359ce6df89"> Also change the workflow so we can use workflow dispatch and so that it has the env vars
pytorch · Jul 9, 2024 · 00e63a8 · 00e63a8
1 parent 0f0ff39
commit 00e63a8
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/.github/workflows/update-queue-times.yml b/.github/workflows/update-queue-times.yml
@@ -4,6 +4,7 @@ on:
   schedule:
     # Run every 15 minutes
     - cron: "*/15 * * * *"
+  workflow_dispatch:
 
 defaults:
   run:
@@ -24,4 +25,6 @@ jobs:
           aws-region: us-east-1
       - run: yarn node scripts/updateQueueTimes.mjs
         env:
+          OUR_AWS_ACCESS_KEY_ID: ${{ steps.aws_creds.outputs.aws-access-key-id }}
+          OUR_AWS_SECRET_ACCESS_KEY: ${{ steps.aws_creds.outputs.aws-secret-access-key }}
           ROCKSET_API_KEY: ${{ secrets.ROCKSET_API_KEY }}
diff --git a/torchci/rockset/metrics/__sql/queue_times_historical.sql b/torchci/rockset/metrics/__sql/queue_times_historical.sql
@@ -2,7 +2,7 @@ SELECT
     FORMAT_ISO8601(
         DATE_TRUNC(
             :granularity,
-            q._event_time AT TIME ZONE :timezone
+            q.time AT TIME ZONE :timezone
         )
     ) AS granularity_bucket,
     /* misnomer, this is the max queue time, not the avg queue time */
@@ -11,8 +11,8 @@ SELECT
 FROM
     metrics.queue_times_historical q
 WHERE
-    q._event_time >= PARSE_DATETIME_ISO8601(:startTime) AT TIME ZONE :timezone
-    AND q._event_time < PARSE_DATETIME_ISO8601(:stopTime) AT TIME ZONE :timezone
+    q.time >= PARSE_DATETIME_ISO8601(:startTime) AT TIME ZONE :timezone
+    AND q.time < PARSE_DATETIME_ISO8601(:stopTime) AT TIME ZONE :timezone
 GROUP BY
     granularity_bucket,
     q.machine_type

diff --git a/torchci/rockset/prodVersions.json b/torchci/rockset/prodVersions.json
@@ -73,7 +73,7 @@
     "tts_duration_historical": "88c02c6e25d59854",
     "tts_duration_historical_percentile": "f6824cbe03e1b6d8",
     "strict_lag_sec": "e0ab723990d6f2a2",
-    "queue_times_historical": "f4a5f7cab1f00b24",
+    "queue_times_historical": "2069a7304a266f6c",
     "workflow_duration_avg": "7bae00900097a486",
     "workflow_duration_percentile": "0b28cf65de7cfe07",
     "workflow_load": "c4e5806c20e64d01",