feat: support visualizing inference metrics

package.json

@@ -36,6 +36,7 @@
     "theme-schema:update": "pnpm dlx typescript-json-schema \"./react/node_modules/antd/es/config-provider/context.d.ts\" ThemeConfig -o ./resources/antdThemeConfig.schema.json --esModuleInterop"
   },
   "dependencies": {
+    "@ant-design/charts": "^2.2.3",
     "@iarna/toml": "^2.2.5",
     "@lit/reactive-element": "^2.0.4",
     "@material/mwc-button": "^0.27.0",

react/data/schema.graphql

@@ -411,7 +411,7 @@ type DomainNode implements Node {
   scaling_groups(filter: String, order: String, offset: Int, before: String, after: String, first: Int, last: Int): ScalinGroupConnection
 }
 
-"""Added in 24.12.0."""
+"""Added in 24.09.1."""
 scalar Bytes
 
 """Added in 24.12.0."""
@@ -1373,6 +1373,7 @@ type Endpoint implements Item {
   status: String
   lifecycle_stage: String
   errors: [InferenceSessionError!]!
+  live_stat: JSONString
 }
 
 """Added in 24.03.5."""
@@ -1390,6 +1391,7 @@ type Routing implements Item {
   traffic_ratio: Float
   created_at: DateTime
   error_data: JSONString
+  live_stat: JSONString
 }
 
 type InferenceSessionError {
@@ -2592,4 +2594,4 @@ type CheckAndTransitStatus {
 input CheckAndTransitStatusInput {
   ids: [GlobalIDField]!
   client_mutation_id: String
-}
+}

react/src/components/EndpointMetricsModal.tsx

@@ -0,0 +1,186 @@
+import BAIModal, { BAIModalProps } from './BAIModal';
+import { EndpointMetricsModalQuery } from './__generated__/EndpointMetricsModalQuery.graphql';
+import { Bar, Line } from '@ant-design/charts';
+import { DownOutlined } from '@ant-design/icons';
+import { Col, Dropdown, Row, Space } from 'antd';
+import graphql from 'babel-plugin-relay/macro';
+import _ from 'lodash';
+import React, { useEffect, useMemo, useRef, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import {
+  fetchQuery,
+  useFragment,
+  useLazyLoadQuery,
+  useMutation,
+  useRelayEnvironment,
+} from 'react-relay';
+
+interface EndpointMetricsModalProps extends BAIModalProps {
+  endpoint_id: string;
+}
+
+type LineChartData = {
+  label: string;
+  data: number;
+};
+
+const EndpointMetrics: React.FC<{ endpoint_id: string }> = ({
+  endpoint_id,
+}: {
+  endpoint_id: string;
+}) => {
+  const relayEvn = useRelayEnvironment();
+
+  const [dataSeries, setDataSeries] =
+    useState<{ liveStat: any; lastUpdatedAt: Date }[]>();
+  const [selectedMetric, setSelectedMetric] = useState<string>();
+  const [chartType, setChartType] = useState<string>();
+
+  const updateStatInfo = async () => {
+    const response = await fetchQuery<EndpointMetricsModalQuery>(
+      relayEvn,
+      graphql`
+        query EndpointMetricsModalQuery($endpointId: UUID!) {
+          endpoint(endpoint_id: $endpointId) {
+            name
+            live_stat @since(version: "24.03.11")
+          }
+        }
+      `,
+      { endpointId: endpoint_id },
+    ).toPromise();
+    const liveStatJSON = response?.endpoint?.live_stat;
+    if (liveStatJSON) {
+      setDataSeries((prev) =>
+        [
+          ...(prev || []),
+          { liveStat: JSON.parse(liveStatJSON), lastUpdatedAt: new Date() },
+        ].slice(-30),
+      );
+    }
+  };
+
+  useEffect(() => {
+    const itv = setInterval(updateStatInfo, 1500);
+    updateStatInfo();
+
+    return () => clearInterval(itv);
+  }, []);
+
+  const chartData = useMemo(() => {
+    if (!dataSeries || !selectedMetric) return;
+
+    const dataPoints = dataSeries.map((data) => data.liveStat[selectedMetric]);
+    if (dataPoints.length === 0) return;
+    const dataType = dataPoints[0].__type;
+
+    setChartType(dataType);
+    switch (dataType) {
+      case 'HISTOGRAM':
+        const cdf = dataPoints[dataPoints.length - 1].current;
+        const max = parseFloat(cdf['+Inf']);
+        return Array.from(Object.keys(cdf)).map((label) => ({
+          label,
+          data: parseFloat(cdf[label]) / max,
+        }));
+      case 'GAUGE':
+      case 'ACCUMULATION':
+        return dataPoints.map((point, index) => {
+          const label = `${_.padStart(dataSeries[index].lastUpdatedAt.getHours().toString(), 2, '0')}:${_.padStart(dataSeries[index].lastUpdatedAt.getMinutes().toString(), 2, '0')}:${_.padStart(dataSeries[index].lastUpdatedAt.getSeconds().toString(), 2, '0')}`;
+          return { label, data: parseFloat(point.current) };
+        });
+      default:
+        return;
+    }
+  }, [dataSeries, selectedMetric]);
+
+  const chartStyles = useMemo(() => {
+    if (!chartType) return;
+
+    switch (chartType) {
+      case 'HISTOGRAM':
+        return {
+          point: {
+            sizeField: 4,
+          },
+          style: {
+            lineWidth: 2,
+            stroke: 'red',
+          },
+          animation: false,
+          animate: false,
+          autoFit: true,
+        };
+      case 'GAUGE':
+      case 'ACCUMULATION':
+        return {
+          style: {
+            lineWidth: 2,
+            stroke: 'blue',
+          },
+          animation: false,
+          animate: false,
+          autoFit: true,
+        };
+    }
+  }, [chartType]);
+
+  return (
+    <div>
+      <Dropdown
+        menu={{
+          items: dataSeries
+            ? Array.from(Object.keys(dataSeries[0].liveStat)).map((key) => ({
+                key,
+                label: `${key} (${dataSeries[0].liveStat[key].__type})`,
+              }))
+            : [],
+          onClick: (info) => setSelectedMetric(info.key),
+        }}
+        disabled={dataSeries?.length === 0}
+      >
+        <a onClick={(e) => e.preventDefault()}>
+          <Space>
+            {selectedMetric
+              ? `${selectedMetric} (${chartType})`
+              : 'Select Metric'}
+            <DownOutlined />
+          </Space>
+        </a>
+      </Dropdown>
+
+      <div style={{ height: '80%' }}>
+        {selectedMetric && chartData ? (
+          <Line
+            data={chartData}
+            xField="label"
+            yField="data"
+            title={selectedMetric}
+            {...(chartStyles || {})}
+          />
+        ) : null}
+      </div>
+    </div>
+  );
+};
+
+const EndpointMetricsModal: React.FC<EndpointMetricsModalProps> = ({
+  endpoint_id,
+  ...modalProps
+}) => {
+  const { t } = useTranslation();
+
+  return (
+    <BAIModal
+      {...modalProps}
+      footer={null}
+      title={t('button.ViewMetrics')}
+      destroyOnClose
+      width="90%"
+    >
+      <EndpointMetrics endpoint_id={endpoint_id} />
+    </BAIModal>
+  );
+};
+
+export default EndpointMetricsModal;

react/src/pages/EndpointDetailPage.tsx

@@ -1,4 +1,5 @@
 import CopyableCodeText from '../components/CopyableCodeText';
+import EndpointMetricsModal from '../components/EndpointMetricsModal';
 import EndpointOwnerInfo from '../components/EndpointOwnerInfo';
 import EndpointStatusTag from '../components/EndpointStatusTag';
 import EndpointTokenGenerationModal from '../components/EndpointTokenGenerationModal';
@@ -29,6 +30,7 @@ import {
   CheckOutlined,
   CloseOutlined,
   FolderOutlined,
+  LineChartOutlined,
   LoadingOutlined,
   PlusOutlined,
   QuestionCircleOutlined,
@@ -102,6 +104,8 @@ const EndpointDetailPage: React.FC<EndpointDetailPageProps> = () => {
   const [isPendingClearError, startClearErrorTransition] = useTransition();
   const [selectedSessionErrorForModal, setSelectedSessionErrorForModal] =
     useState<InferenceSessionErrorModalFragment$key | null>(null);
+  const [openEndpointMetricsModal, setOpenEndpointMetricsModal] =
+    useState(false);
   const [isOpenTokenGenerationModal, setIsOpenTokenGenerationModal] =
     useState(false);
   const [openChatModal, setOpenChatModal] = useState(false);
@@ -182,6 +186,7 @@ const EndpointDetailPage: React.FC<EndpointDetailPageProps> = () => {
               status
             }
             created_user_email @since(version: "23.09.8")
+            live_stat @since(version: "23.09.11")
             ...EndpointOwnerInfoFragment
             ...EndpointStatusTagFragment
             ...ChatUIModalFragment
@@ -476,6 +481,16 @@ const EndpointDetailPage: React.FC<EndpointDetailPageProps> = () => {
           ) : (
             <></>
           )}
+          <Button
+            loading={isPendingRefetch}
+            disabled={!endpoint?.live_stat}
+            icon={<LineChartOutlined />}
+            onClick={() => {
+              setOpenEndpointMetricsModal(true);
+            }}
+          >
+            {t('button.ViewMetrics')}
+          </Button>
           <Button
             loading={isPendingRefetch}
             icon={<ReloadOutlined />}
@@ -728,6 +743,13 @@ const EndpointDetailPage: React.FC<EndpointDetailPageProps> = () => {
           setOpenChatModal(false);
         }}
       />
+      <EndpointMetricsModal
+        endpoint_id={endpoint?.endpoint_id || ''}
+        open={openEndpointMetricsModal}
+        onCancel={() => {
+          setOpenEndpointMetricsModal(false);
+        }}
+      />
       <SessionDetailDrawer
         open={!selectedSessionId}
         sessionId={selectedSessionId}

resources/i18n/en.json

@@ -563,7 +563,8 @@
     "Expand": "Expand",
     "Clear": "Clear",
     "Apply": "Apply",
-    "CopySomething": "Copy {{name}}"
+    "CopySomething": "Copy {{name}}",
+    "ViewMetrics": "View Metrics"
   },
   "agent": {
     "Endpoint": "Endpoint",

version.json

@@ -1 +1 @@
-{ "package": "24.09.0-alpha.1", "buildNumber": "6111", "buildDate": "240703.130731", "revision": "d8cea1cb" }
+{ "package": "24.09.0-alpha.1", "buildNumber": "6357", "buildDate": "241119.011126", "revision": "90496bcf" }