[ch][hud metrics] workflow_load (#5638)

* put utc everywhere to handle the lack of "Z" * I think its ok to not take timezone into account when calculating the granularity bucket * the type casting is getting a bit annoying, I'm considering just switching to CH at some point and calling it a day
pytorch · Sep 11, 2024 · d7af4ff · d7af4ff
1 parent 46d04a3
commit d7af4ff
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 38 deletions.
diff --git a/torchci/clickhouse_queries/workflow_load/query.sql b/torchci/clickhouse_queries/workflow_load/query.sql
@@ -1,18 +1,22 @@
--- !!! Query is not converted to CH syntax yet.  Delete this line when it gets converted
 SELECT
-    FORMAT_ISO8601(
-        DATE_TRUNC(
-            :granularity,
-            PARSE_TIMESTAMP_ISO8601(workflow.created_at) AT TIME ZONE :timezone
-        )
+    DATE_TRUNC(
+        {granularity: String},
+        workflow.created_at
     ) AS granularity_bucket,
     workflow.name,
-    COUNT(*) as count,
+    COUNT(*) as count
 FROM
-    workflow_run workflow
+    default.workflow_run workflow final
 WHERE
-    PARSE_TIMESTAMP_ISO8601(workflow.created_at) >= PARSE_DATETIME_ISO8601(:startTime)
-    AND PARSE_TIMESTAMP_ISO8601(workflow.created_at) < PARSE_DATETIME_ISO8601(:stopTime)
+    -- optimization to make query faster
+    workflow.id in (
+        select id from materialized_views.workflow_run_by_created_at
+        where created_at >= {startTime: DateTime64(9)}
+        AND created_at <= {stopTime: DateTime64(9)}
+    )
+    -- re check for final
+    and workflow.created_at >= {startTime: DateTime64(9)}
+    AND workflow.created_at <= {stopTime: DateTime64(9)}
     AND workflow.name IN (
         'pull',
         'trunk',
@@ -26,7 +30,7 @@ WHERE
         'rocm',
         'inductor-rocm'
     )
-    AND workflow.repository.full_name like :repo
+    AND workflow.repository.'full_name' like {repo: String}
 GROUP BY
     granularity_bucket,
     workflow.name

diff --git a/torchci/components/metrics/panels/TimeSeriesPanel.tsx b/torchci/components/metrics/panels/TimeSeriesPanel.tsx
@@ -48,7 +48,7 @@ export function seriesWithInterpolatedTimes(
     t = startTime;
   for (let i = 0; t.isBefore(stopTime) && i < times.length; i++) {
     prevT = t;
-    t = dayjs(times[i]);
+    t = dayjs.utc(times[i]);
 
     let timeGap = t.diff(prevT, granularity);
     if (timeGap > 1.15) {
@@ -75,7 +75,7 @@ export function seriesWithInterpolatedTimes(
     const byTime = _.keyBy(value, timeFieldName);
     // Roundtrip each timestamp to make the format uniform.
     const byTimeNormalized = _.mapKeys(byTime, (_, k) =>
-      dayjs(k).toISOString()
+      dayjs.utc(k).toISOString()
     );
 
     // Fill with 0, see the above comment on interpolation.
@@ -176,7 +176,8 @@ export function TimeSeriesPanelWithData({
         trigger: "item",
         formatter: (params: any) =>
           `${params.seriesName}` +
-          `<br/>${dayjs(params.value[0])
+          `<br/>${dayjs
+            .utc(params.value[0])
             .local()
             .format(timeFieldDisplayFormat)}<br/>` +
           `${getTooltipMarker(params.color)}` +
@@ -222,11 +223,12 @@ export default function TimeSeriesPanel({
   yAxisLabel,
   // Additional EChartsOption (ex max y value)
   additionalOptions,
+  useClickHouse = false,
 }: {
   title: string;
   queryCollection?: string;
   queryName: string;
-  queryParams: RocksetParam[];
+  queryParams: RocksetParam[] | {};
   granularity: Granularity;
   groupByFieldName?: string;
   timeFieldName: string;
@@ -235,16 +237,24 @@ export default function TimeSeriesPanel({
   yAxisRenderer: (_value: any) => string;
   yAxisLabel?: string;
   additionalOptions?: EChartsOption;
+  useClickHouse?: boolean;
 }) {
   // - Granularity
   // - Group by
   // - Time field
-  const url = `/api/query/${queryCollection}/${queryName}?parameters=${encodeURIComponent(
-    JSON.stringify([
-      ...queryParams,
-      { name: "granularity", type: "string", value: granularity },
-    ])
-  )}`;
+  const url = useClickHouse
+    ? `/api/clickhouse/${queryName}?parameters=${encodeURIComponent(
+        JSON.stringify({
+          ...(queryParams as {}),
+          granularity: granularity as string,
+        })
+      )}`
+    : `/api/query/${queryCollection}/${queryName}?parameters=${encodeURIComponent(
+        JSON.stringify([
+          ...(queryParams as RocksetParam[]),
+          { name: "granularity", type: "string", value: granularity },
+        ])
+      )}`;
 
   const { data } = useSWR(url, fetcher, {
     refreshInterval: 5 * 60 * 1000, // refresh every 5 minutes
@@ -254,13 +264,23 @@ export default function TimeSeriesPanel({
     return <Skeleton variant={"rectangular"} height={"100%"} />;
   }
 
-  let startTime = queryParams.find((p) => p.name === "startTime")?.value;
-  let stopTime = queryParams.find((p) => p.name === "stopTime")?.value;
+  let startTime, stopTime;
+  if (useClickHouse) {
+    startTime = (queryParams as any)["startTime"];
+    stopTime = (queryParams as any)["stopTime"];
+  } else {
+    startTime = (queryParams as RocksetParam[]).find(
+      (p) => p.name === "startTime"
+    )?.value;
+    stopTime = (queryParams as RocksetParam[]).find(
+      (p) => p.name === "stopTime"
+    )?.value;
+  }
 
   // Clamp to the nearest granularity (e.g. nearest hour) so that the times will
   // align with the data we get from Rockset
-  startTime = dayjs(startTime).startOf(granularity);
-  stopTime = dayjs(stopTime).endOf(granularity);
+  startTime = dayjs.utc(startTime).startOf(granularity);
+  stopTime = dayjs.utc(stopTime).endOf(granularity);
 
   const series = seriesWithInterpolatedTimes(
     data,

diff --git a/torchci/pages/metrics.tsx b/torchci/pages/metrics.tsx
@@ -957,25 +957,30 @@ export default function Page() {
           <TimeSeriesPanel
             title={"Workflow load per Day"}
             queryName={"workflow_load"}
-            queryParams={[
-              {
-                name: "timezone",
-                type: "string",
-                value: Intl.DateTimeFormat().resolvedOptions().timeZone,
-              },
-              {
-                name: "repo",
-                type: "string",
-                value: "pytorch/pytorch",
-              },
-              ...timeParams,
-            ]}
+            queryParams={
+              useClickHouse
+                ? { ...timeParamsClickHouse, repo: "pytorch/pytorch" }
+                : [
+                    {
+                      name: "timezone",
+                      type: "string",
+                      value: Intl.DateTimeFormat().resolvedOptions().timeZone,
+                    },
+                    {
+                      name: "repo",
+                      type: "string",
+                      value: "pytorch/pytorch",
+                    },
+                    ...timeParams,
+                  ]
+            }
             granularity={"hour"}
             groupByFieldName={"name"}
             timeFieldName={"granularity_bucket"}
             yAxisFieldName={"count"}
             yAxisLabel={"workflows started"}
             yAxisRenderer={(value) => value}
+            useClickHouse={useClickHouse}
           />
         </Grid>