feat(weave): Small-batch, on-demand LLM Judge

wb_schema.gql

@@ -169,6 +169,7 @@ type User implements Node {
   photoUrl: String
   deletedAt: DateTime
   teams(before: String, after: String, first: Int, last: Int): EntityConnection
+  admin: Boolean
 }
 
 type UserConnection {

weave-js/src/common/hooks/useViewerInfo.ts

@@ -11,6 +11,7 @@ const VIEWER_QUERY = gql`
     viewer {
       id
       username
+      admin
       teams {
         edges {
           node {
@@ -28,6 +29,7 @@ type UserInfo = {
   id: string;
   username: string;
   teams: string[];
+  admin: boolean;
 };
 type UserInfoResponseLoading = {
   loading: true;
@@ -71,6 +73,7 @@ export const useViewerInfo = (): UserInfoResponse => {
           id,
           username,
           teams,
+          admin: userInfo.admin,
         },
       });
     });

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/FeedbackGrid.tsx

@@ -1,12 +1,13 @@
 import {Box} from '@mui/material';
 import _ from 'lodash';
-import React, {useEffect} from 'react';
+import React, {useEffect, useMemo} from 'react';
 
 import {useViewerInfo} from '../../../../../common/hooks/useViewerInfo';
 import {TargetBlank} from '../../../../../common/util/links';
 import {Alert} from '../../../../Alert';
 import {Loading} from '../../../../Loading';
 import {Tailwind} from '../../../../Tailwind';
+import {RUNNABLE_FEEDBACK_TYPE_PREFIX} from '../pages/CallPage/CallScoresViewer';
 import {Empty} from '../pages/common/Empty';
 import {useWFHooks} from '../pages/wfReactInterface/context';
 import {useGetTraceServerClientContext} from '../pages/wfReactInterface/traceServerClientContext';
@@ -40,6 +41,23 @@ export const FeedbackGrid = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
+  // Exclude runnables as they are presented in a different tab
+  const withoutRunnables = useMemo(
+    () =>
+      (query.result ?? []).filter(
+        f => !f.feedback_type.startsWith(RUNNABLE_FEEDBACK_TYPE_PREFIX)
+      ),
+    [query.result]
+  );
+
+  // Group by feedback on this object vs. descendent objects
+  const grouped = useMemo(
+    () =>
+      _.groupBy(withoutRunnables, f => f.weave_ref.substring(weaveRef.length)),
+    [withoutRunnables, weaveRef]
+  );
+  const paths = useMemo(() => Object.keys(grouped).sort(), [grouped]);
+
   if (query.loading || loadingUserInfo) {
     return (
       <Box
@@ -61,7 +79,7 @@ export const FeedbackGrid = ({
     );
   }
 
-  if (!query.result || !query.result.length) {
+  if (!withoutRunnables.length) {
     return (
       <Empty
         size="small"
@@ -81,12 +99,6 @@ export const FeedbackGrid = ({
     );
   }
 
-  // Group by feedback on this object vs. descendent objects
-  const grouped = _.groupBy(query.result, f =>
-    f.weave_ref.substring(weaveRef.length)
-  );
-  const paths = Object.keys(grouped).sort();
-
   const currentViewerId = userInfo ? userInfo.id : null;
   return (
     <Tailwind>

weave-js/src/components/PagePanelComponents/Home/Browse3/pages/CallPage/CallPage.tsx

@@ -1,4 +1,5 @@
 import Box from '@mui/material/Box';
+import {useViewerInfo} from '@wandb/weave/common/hooks/useViewerInfo';
 import {Loading} from '@wandb/weave/components/Loading';
 import {useViewTraceEvent} from '@wandb/weave/integrations/analytics/useViewEvents';
 import React, {FC, useCallback, useContext, useEffect, useState} from 'react';
@@ -31,6 +32,7 @@ import {CallSchema} from '../wfReactInterface/wfDataModelHooksInterface';
 import {CallChat} from './CallChat';
 import {CallDetails} from './CallDetails';
 import {CallOverview} from './CallOverview';
+import {CallScoresViewer} from './CallScoresViewer';
 import {CallSummary} from './CallSummary';
 import {CallTraceView, useCallFlattenedTraceTree} from './CallTraceView';
 import {PaginationControls} from './PaginationControls';
@@ -56,7 +58,13 @@ export const CallPage: FC<{
   return <CallPageInnerVertical {...props} call={call.result} />;
 };
 
+export const useShowRunnableUI = () => {
+  const viewerInfo = useViewerInfo();
+  return viewerInfo.loading ? false : viewerInfo.userInfo?.admin;
+};
+
 const useCallTabs = (call: CallSchema) => {
+  const showScores = useShowRunnableUI();
   const codeURI = call.opVersionRef;
   const {entity, project, callId} = call;
   const weaveRef = makeRefCall(entity, project, callId);
@@ -128,6 +136,21 @@ const useCallTabs = (call: CallSchema) => {
         </Tailwind>
       ),
     },
+    // For now, we are only showing this tab for W&B admins since the
+    // feature is in active development. We want to be able to get
+    // feedback without enabling for all users.
+    ...(showScores
+      ? [
+          {
+            label: 'Scores (W&B Admin Preview)',
+            content: (
+              <Tailwind>
+                <CallScoresViewer call={call} />
+              </Tailwind>
+            ),
+          },
+        ]
+      : []),
     {
       label: 'Use',
       content: (