feat(weave): human feedback sidebar for structured call labeling

weave-js/src/components/PagePanelComponents/Home/Browse3/context.tsx

@@ -336,7 +336,8 @@ export const browse3ContextGen = (
       traceId: string,
       callId: string,
       path?: string | null,
-      tracetree?: boolean
+      tracetree?: boolean,
+      feedbackExpand?: boolean
     ) => {
       let url = `${projectRoot(entityName, projectName)}/calls/${callId}`;
       const params = new URLSearchParams();
@@ -346,6 +347,9 @@ export const browse3ContextGen = (
       if (tracetree !== undefined) {
         params.set(TRACETREE_PARAM, tracetree ? '1' : '0');
       }
+      if (feedbackExpand !== undefined) {
+        params.set(FEEDBACK_EXPAND_PARAM, feedbackExpand ? '1' : '0');
+      }
       if (params.toString()) {
         url += '?' + params.toString();
       }
@@ -497,7 +501,8 @@ type RouteType = {
     traceId: string,
     callId: string,
     path?: string | null,
-    tracetree?: boolean
+    tracetree?: boolean,
+    feedbackExpand?: boolean
   ) => string;
   tracesUIUrl: (entityName: string, projectName: string) => string;
   callsUIUrl: (
@@ -564,6 +569,7 @@ const useSetSearchParam = () => {
 
 export const PEEK_PARAM = 'peekPath';
 export const TRACETREE_PARAM = 'tracetree';
+export const FEEDBACK_EXPAND_PARAM = 'feedbackExpand';
 export const PATH_PARAM = 'path';
 
 export const baseContext = browse3ContextGen(

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/FeedbackGrid.tsx

@@ -11,6 +11,9 @@ import {Empty} from '../pages/common/Empty';
 import {useWFHooks} from '../pages/wfReactInterface/context';
 import {useGetTraceServerClientContext} from '../pages/wfReactInterface/traceServerClientContext';
 import {FeedbackGridInner} from './FeedbackGridInner';
+import {HUMAN_ANNOTATION_BASE_TYPE} from './StructuredFeedback/humanAnnotationTypes';
+
+const ANNOTATION_PREFIX = `${HUMAN_ANNOTATION_BASE_TYPE}.`;
 
 type FeedbackGridProps = {
   entity: string;
@@ -81,8 +84,22 @@ export const FeedbackGrid = ({
     );
   }
 
+  // Combine annotation feedback on (feedback_type, creator)
+  const combined = _.groupBy(
+    query.result.filter(f => f.feedback_type.startsWith(ANNOTATION_PREFIX)),
+    f => `${f.feedback_type}-${f.creator}`
+  );
+  // only keep the most recent feedback for each (feedback_type, creator)
+  const combinedFiltered = Object.values(combined).map(
+    fs => fs.sort((a, b) => b.created_at - a.created_at)[0]
+  );
+  // add the non-annotation feedback to the combined object
+  combinedFiltered.push(
+    ...query.result.filter(f => !f.feedback_type.startsWith(ANNOTATION_PREFIX))
+  );
+
   // Group by feedback on this object vs. descendent objects
-  const grouped = _.groupBy(query.result, f =>
+  const grouped = _.groupBy(combinedFiltered, f =>
     f.weave_ref.substring(weaveRef.length)
   );
   const paths = Object.keys(grouped).sort();

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/FeedbackGridInner.tsx

@@ -42,6 +42,13 @@ export const FeedbackGridInner = ({
         if (params.row.feedback_type === 'wandb.reaction.1') {
           return params.row.payload.emoji;
         }
+        if (params.row.feedback_type.startsWith('wandb.annotation.')) {
+          return (
+            <CellValueString
+              value={JSON.stringify(params.row.payload.value ?? null)}
+            />
+          );
+        }
         return <CellValueString value={JSON.stringify(params.row.payload)} />;
       },
     },

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/FeedbackTypeChip.tsx

@@ -15,6 +15,9 @@ export const FeedbackTypeChip = ({feedbackType}: FeedbackTypeChipProps) => {
   } else if (feedbackType === 'wandb.note.1') {
     color = 'gold';
     label = 'Note';
+  } else if (feedbackType.includes('wandb.annotation.')) {
+    color = 'magenta';
+    label = 'Annotation';
   }
   return <Pill color={color} label={label} />;
 };

weave-js/src/components/PagePanelComponents/Home/Browse3/feedback/StructuredFeedback/FeedbackSidebar.tsx

@@ -0,0 +1,248 @@
+import {toast} from '@wandb/weave/common/components/elements/Toast';
+import {useViewerInfo} from '@wandb/weave/common/hooks/useViewerInfo';
+import {Button} from '@wandb/weave/components/Button';
+import {Icon} from '@wandb/weave/components/Icon';
+import {makeRefCall} from '@wandb/weave/util/refs';
+import React, {useState} from 'react';
+import {useHistory} from 'react-router-dom';
+
+import {useWeaveflowRouteContext} from '../../context';
+import {Empty} from '../../pages/common/Empty';
+import {EMPTY_PROPS_ANNOTATIONS} from '../../pages/common/EmptyContent';
+import {HumanAnnotationCell} from './HumanAnnotation';
+import {tsHumanAnnotationSpec} from './humanAnnotationTypes';
+
+type FeedbackSidebarProps = {
+  humanAnnotationSpecs: tsHumanAnnotationSpec[];
+  callID: string;
+  entity: string;
+  project: string;
+};
+
+export const FeedbackSidebar = ({
+  humanAnnotationSpecs,
+  callID,
+  entity,
+  project,
+}: FeedbackSidebarProps) => {
+  const history = useHistory();
+  const router = useWeaveflowRouteContext().baseRouter;
+  const [isSaving, setIsSaving] = useState(false);
+  const [unsavedFeedbackChanges, setUnsavedFeedbackChanges] = useState<
+    Record<string, () => Promise<boolean>>
+  >({});
+
+  const save = async () => {
+    setIsSaving(true);
+    try {
+      // Save all pending feedback changes
+      const savePromises = Object.values(unsavedFeedbackChanges).map(saveFn =>
+        saveFn()
+      );
+      const results = await Promise.all(savePromises);
+
+      // Check if any saves failed
+      if (results.some(result => !result)) {
+        throw new Error('Not all feedback changes saved');
+      }
+
+      // Clear the unsaved changes after successful save
+      setUnsavedFeedbackChanges({});
+    } catch (error) {
+      console.error('Error saving feedback:', error);
+      toast(`Error saving feedback: ${error}`, {
+        type: 'error',
+      });
+    } finally {
+      setIsSaving(false);
+    }
+  };
+
+  return (
+    <div className="flex h-full flex-col bg-white">
+      <div className="justify-left flex w-full border-b border-moon-300 p-12">
+        <div className="text-lg font-semibold">Feedback</div>
+        <div className="flex-grow" />
+      </div>
+      {humanAnnotationSpecs.length > 0 ? (
+        <>
+          <div className="mx-6 h-full flex-grow overflow-auto">
+            <HumanAnnotationSection
+              entity={entity}
+              project={project}
+              callID={callID}
+              humanAnnotationSpecs={humanAnnotationSpecs}
+              setUnsavedFeedbackChanges={setUnsavedFeedbackChanges}
+            />
+          </div>
+          <div className="flex w-full border-t border-moon-300 p-6 pr-10">
+            <Button
+              onClick={save}
+              variant="primary"
+              className="w-full"
+              disabled={
+                isSaving || Object.keys(unsavedFeedbackChanges).length === 0
+              }
+              size="large">
+              {isSaving ? 'Saving...' : 'Save'}
+            </Button>
+          </div>
+        </>
+      ) : (
+        <div className="mt-12 w-full items-center justify-center">
+          <Empty {...EMPTY_PROPS_ANNOTATIONS} />
+          <div className="mt-4 flex w-full justify-center">
+            <Button
+              onClick={() =>
+                history.push(router.scorersUIUrl(entity, project))
+              }>
+              View scorers
+            </Button>
+          </div>
+        </div>
+      )}
+    </div>
+  );
+};
+
+type HumanAnnotationSectionProps = {
+  entity: string;
+  project: string;
+  callID: string;
+  humanAnnotationSpecs: tsHumanAnnotationSpec[];
+  setUnsavedFeedbackChanges: React.Dispatch<
+    React.SetStateAction<Record<string, () => Promise<boolean>>>
+  >;
+};
+
+const HumanAnnotationSection = ({
+  entity,
+  project,
+  callID,
+  humanAnnotationSpecs,
+  setUnsavedFeedbackChanges,
+}: HumanAnnotationSectionProps) => {
+  const [isExpanded, setIsExpanded] = useState(true);
+  const sortedVisibleColumns = humanAnnotationSpecs.sort((a, b) =>
+    (a.name ?? '').localeCompare(b.name ?? '')
+  );
+
+  return (
+    <div>
+      <HumanAnnotationHeader
+        numHumanAnnotationSpecsVisible={sortedVisibleColumns.length}
+        numHumanAnnotationSpecsHidden={
+          humanAnnotationSpecs.length - sortedVisibleColumns.length
+        }
+        isExpanded={isExpanded}
+        setIsExpanded={setIsExpanded}
+      />
+      {isExpanded && (
+        <HumanAnnotationInputs
+          entity={entity}
+          project={project}
+          callID={callID}
+          humanAnnotationSpecs={sortedVisibleColumns}
+          setUnsavedFeedbackChanges={setUnsavedFeedbackChanges}
+        />
+      )}
+    </div>
+  );
+};
+
+type HumanAnnotationHeaderProps = {
+  numHumanAnnotationSpecsVisible: number;
+  numHumanAnnotationSpecsHidden: number;
+  isExpanded: boolean;
+  setIsExpanded: (isExpanded: boolean) => void;
+};
+
+const HumanAnnotationHeader = ({
+  numHumanAnnotationSpecsVisible,
+  numHumanAnnotationSpecsHidden,
+  isExpanded,
+  setIsExpanded,
+}: HumanAnnotationHeaderProps) => {
+  return (
+    <button
+      className="text-md hover:bg-gray-100 flex w-full items-center justify-between px-6 py-8 font-semibold"
+      onClick={() => setIsExpanded(!isExpanded)}>
+      <div className="mb-8 flex w-full items-center">
+        <div className="text-lg">Human annotations</div>
+        <div className="ml-6 mt-1">
+          <DisplayNumericCounter count={numHumanAnnotationSpecsVisible} />
+        </div>
+        <div className="flex-grow" />
+        {numHumanAnnotationSpecsHidden > 0 && (
+          <div className="mr-4 mt-2 rounded-full px-2 text-xs font-medium">
+            {numHumanAnnotationSpecsHidden} hidden
+          </div>
+        )}
+      </div>
+      <div className="mb-6 flex items-center">
+        <Icon name={isExpanded ? 'chevron-up' : 'chevron-down'} />
+      </div>
+    </button>
+  );
+};
+
+type HumanAnnotationInputsProps = {
+  entity: string;
+  project: string;
+  callID: string;
+  humanAnnotationSpecs: tsHumanAnnotationSpec[];
+  setUnsavedFeedbackChanges: React.Dispatch<
+    React.SetStateAction<Record<string, () => Promise<boolean>>>
+  >;
+};
+
+const HumanAnnotationInputs = ({
+  entity,
+  project,
+  callID,
+  humanAnnotationSpecs,
+  setUnsavedFeedbackChanges,
+}: HumanAnnotationInputsProps) => {
+  const callRef = makeRefCall(entity, project, callID);
+  const {loading: loadingUserInfo, userInfo} = useViewerInfo();
+
+  if (loadingUserInfo) {
+    return null;
+  }
+  const viewer = userInfo ? userInfo.id : null;
+
+  return (
+    <div>
+      {humanAnnotationSpecs?.map((field, index) => (
+        <div key={field.ref} className="px-16">
+          <div className="bg-gray-50 text-md font-semibold">{field.name}</div>
+          {field.description && (
+            <div className="bg-gray-50 font-italic mt-4 text-sm text-moon-700 ">
+              {field.description}
+            </div>
+          )}
+          <div className="pb-8 pt-4">
+            <HumanAnnotationCell
+              focused={index === 0}
+              hfSpec={field}
+              callRef={callRef}
+              entity={entity}
+              project={project}
+              viewer={viewer}
+              readOnly={false}
+              setUnsavedFeedbackChanges={setUnsavedFeedbackChanges}
+            />
+          </div>
+        </div>
+      ))}
+    </div>
+  );
+};
+
+const DisplayNumericCounter = ({count}: {count: number}) => {
+  return (
+    <div className="rounded-sm bg-moon-150 px-2 text-xs font-medium text-moon-500">
+      {count}
+    </div>
+  );
+};