Skip to content

Commit

Permalink
Merge branch 'master' into griffin/feedback-replace-ts
Browse files Browse the repository at this point in the history
  • Loading branch information
gtarpenning authored Nov 8, 2024
2 parents 005a165 + fef544f commit fd0d7b7
Show file tree
Hide file tree
Showing 122 changed files with 1,155 additions and 73 deletions.
Binary file modified docs/docs/guides/core-types/imgs/audio-trace.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/core-types/imgs/cat-pumpkin-trace.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/anthropic_model.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/anthropic_ops.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/anthropic_tool.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/anthropic_trace.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/cerebras_calls.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/cerebras_model.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/cohere_model.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/cohere_ops.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/cohere_trace.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/dspy/dspy_trace.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/dspy/dspy_weave_model_v1.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/dspy/dspy_weave_model_v2.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/dspy/dspy_with_weave_op.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/groq/groq_weave_dasboard.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/groq/groq_weave_model.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/groq/groq_weave_tracing.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file modified docs/docs/guides/integrations/imgs/langchain_eval.png
Binary file modified docs/docs/guides/integrations/imgs/langchain_model.png
Binary file modified docs/docs/guides/integrations/imgs/langchain_trace.png
Binary file modified docs/docs/guides/integrations/imgs/litellm_gif.gif
Binary file modified docs/docs/guides/integrations/imgs/llamaindex_evaluation.png
Binary file modified docs/docs/guides/integrations/imgs/llamaindex_model.png
Binary file modified docs/docs/guides/integrations/imgs/llamaindex_rag.png
Binary file modified docs/docs/guides/integrations/imgs/mistral_model.png
Binary file modified docs/docs/guides/integrations/imgs/mistral_ops.png
Binary file modified docs/docs/guides/integrations/imgs/mistral_trace.png
Binary file modified docs/docs/guides/integrations/imgs/notdiamond/api-keys.png
Binary file modified docs/docs/guides/integrations/imgs/notdiamond/evaluations.png
Binary file modified docs/docs/guides/integrations/imgs/notdiamond/weave-trace.png
Binary file modified docs/docs/guides/integrations/imgs/openai-model.png
Binary file modified docs/docs/guides/integrations/imgs/openai-pokedex.png
Binary file modified docs/docs/guides/integrations/imgs/openai.png
Binary file modified docs/docs/guides/integrations/imgs/simple_llamaindex.png
Binary file modified docs/docs/guides/tracking/imgs/feedback_call_header.png
Binary file modified docs/docs/guides/tracking/imgs/feedback_calls.png
Binary file modified docs/docs/guides/tracking/imgs/feedback_tab.png
Binary file modified docs/docs/guides/tracking/imgs/feedback_use.png
Binary file modified docs/docs/media/codegen/codegen_trace.png
Binary file modified docs/docs/media/codegen/eval_dash.png
Binary file modified docs/docs/media/codegen/eval_trace.png
Binary file modified docs/docs/media/dspy_optimization/1.png
Binary file modified docs/docs/media/dspy_optimization/2.png
Binary file modified docs/docs/media/dspy_optimization/3.png
Binary file modified docs/docs/media/dspy_optimization/4.png
Binary file modified docs/docs/media/dspy_optimization/5.png
Binary file modified docs/docs/media/intro/1.png
Binary file modified docs/docs/media/intro/10.png
Binary file modified docs/docs/media/intro/2.png
Binary file modified docs/docs/media/intro/3.png
Binary file modified docs/docs/media/intro/4.png
Binary file modified docs/docs/media/intro/5.png
Binary file modified docs/docs/media/intro/6.png
Binary file modified docs/docs/media/intro/7.png
Binary file modified docs/docs/media/intro/8.png
Binary file modified docs/docs/media/intro/9.png
Binary file modified docs/docs/media/multi-agent-structured-output/0.png
Binary file modified docs/docs/media/multi-agent-structured-output/1.png
Binary file modified docs/docs/media/multi-agent-structured-output/2.png
Binary file modified docs/docs/media/multi-agent-structured-output/3.png
Binary file modified docs/docs/media/pii/encrypt.png
Binary file modified docs/docs/media/pii/redact.png
Binary file modified docs/docs/media/pii/replace.png
Binary file modified docs/docs/media/summarization/dataset.png
Binary file modified docs/docs/media/summarization/eval_dash.png
Binary file modified docs/docs/media/summarization/model.png
Binary file modified docs/docs/media/summarization/summarization_trace.png
Binary file modified docs/static/img/call_edit_screenshot.png
Binary file modified docs/static/img/docusaurus-social-card.jpg
Binary file modified docs/static/img/docusaurus.png
Binary file modified docs/static/img/evals-hero.png
Binary file modified docs/static/img/export_modal.png
Binary file modified docs/static/img/logo-large-padded.png
Binary file modified docs/static/img/logo-large.png
Binary file modified docs/static/img/screenshots/basic_call.png
Binary file modified docs/static/img/screenshots/calls_filter.png
Binary file modified docs/static/img/screenshots/calls_macro.png
Binary file modified docs/static/img/tutorial-model_invoke3.png
Binary file modified docs/static/img/tutorial-model_re-use.png
Binary file modified docs/static/img/tutorial_trace_1.png
Binary file modified docs/static/img/tutorial_tracing_2_call_uri.png
Binary file modified docs/static/img/tutorial_tracing_2_nested_dinos.png
Binary file modified docs/static/img/weave-hero.png
Binary file modified docs/static/weave-ui-example.jpg
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ test = [
"clickhouse_connect==0.7.0",
"fastapi>=0.110.0",
"sqlparse==0.5.0",
"jsonschema>=4.23.0",

# Integration Tests
"pytest-recording>=0.13.2",
Expand Down
6 changes: 0 additions & 6 deletions scripts/strip_exif.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,8 @@

def strip_exif(image_path: Path) -> bool:
"""Strip EXIF data from an image file if present.
Args:
image_path: Path to the image file
Returns:
bool: True if successful, False if failed
"""
Expand All @@ -21,10 +19,6 @@ def strip_exif(image_path: Path) -> bool:
if not img.getexif():
return True

# Convert RGBA to RGB if needed (some formats don't support RGBA)
if img.mode == "RGBA":
img = img.convert("RGB")

# Create a new image without EXIF
data = list(img.getdata())
img_without_exif = Image.new(img.mode, img.size)
Expand Down
Binary file modified sdks/node/logs.png
10 changes: 5 additions & 5 deletions tests/integrations/litellm/test_actions_lifecycle_llm_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from tests.trace.util import client_is_sqlite
from weave.trace.weave_client import WeaveClient
from weave.trace_server.interface.base_object_classes.actions import (
ActionDefinition,
ActionSpec,
)
from weave.trace_server.trace_server_interface import (
ActionsExecuteBatchReq,
Expand Down Expand Up @@ -69,9 +69,9 @@ def test_action_lifecycle_llm_judge_primitive(client: WeaveClient):
action_name = "response_is_mindful"

published_ref = weave.publish(
ActionDefinition(
ActionSpec(
name=action_name,
spec={
config={
"action_type": "llm_judge",
"model": "gpt-4o-mini",
"prompt": "Is the response mindful?",
Expand Down Expand Up @@ -154,9 +154,9 @@ def test_action_lifecycle_llm_judge_structured(client: WeaveClient):
action_name = "response_is_mindful"

published_ref = weave.publish(
ActionDefinition(
ActionSpec(
name=action_name,
spec={
config={
"action_type": "llm_judge",
"model": "gpt-4o-mini",
"prompt": "Is the response mindful?",
Expand Down
6 changes: 3 additions & 3 deletions tests/trace/test_actions_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from tests.trace.util import client_is_sqlite
from weave.trace.weave_client import WeaveClient
from weave.trace_server.interface.base_object_classes.actions import (
ActionDefinition,
ActionSpec,
)
from weave.trace_server.trace_server_interface import (
ActionsExecuteBatchReq,
Expand All @@ -22,9 +22,9 @@ def test_action_lifecycle_word_count(client: WeaveClient):
action_name = "my_contains_words_action"

published_ref = weave.publish(
ActionDefinition(
ActionSpec(
name=action_name,
spec={
config={
"action_type": "contains_words",
"target_words": ["mindful", "demure"],
},
Expand Down
58 changes: 58 additions & 0 deletions tests/trace/test_annotation_feedback.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
from weave.flow.annotation_spec import AnnotationSpec
from weave.trace_server.trace_server_interface import ObjQueryReq


def test_human_feedback_basic(client):
# create a human feedback spec

col1 = AnnotationSpec(
name="Numerical field #1",
description="A numerical field with a range of -1 to 1",
json_schema={
"type": "number",
"min": -1,
"max": 1,
},
unique_among_creators=True,
op_scope=None,
)
ref1 = client.save(col1, "my numerical spec")
assert ref1

col2 = AnnotationSpec(
name="Text field #1",
json_schema={"type": "string", "max_length": 100},
op_scope=["weave:///entity/project/op/name:digest"],
)
ref2 = client.save(col2, "my text spec")
assert ref2

# query it by object type
objects = client.server.objs_query(
ObjQueryReq.model_validate(
{
"project_id": client._project_id(),
"filter": {"base_object_classes": ["AnnotationSpec"]},
}
)
)

assert len(objects.objs) == 2
assert objects.objs[0].val["name"] == "Numerical field #1"
assert objects.objs[1].val["name"] == "Text field #1"
assert (
objects.objs[0].val["description"]
== "A numerical field with a range of -1 to 1"
)
assert not objects.objs[1].val["description"]
assert not objects.objs[0].val["op_scope"]
assert objects.objs[1].val["op_scope"] == ["weave:///entity/project/op/name:digest"]
assert objects.objs[0].val["json_schema"] == {
"type": "number",
"min": -1,
"max": 1,
}
assert objects.objs[1].val["json_schema"] == {
"type": "string",
"max_length": 100,
}
16 changes: 16 additions & 0 deletions tests/trace/test_custom_objs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from PIL import Image

from weave.trace.custom_objs import decode_custom_obj, encode_custom_obj


def test_decode_custom_obj_known_type(client):
img = Image.new("RGB", (100, 100))
encoded = encode_custom_obj(img)

# Even though something is wrong with the deserializer op, we can still decode
decoded = decode_custom_obj(
encoded["weave_type"], encoded["files"], "weave:///totally/invalid/uri"
)

assert isinstance(decoded, Image.Image)
assert decoded.tobytes() == img.tobytes()
Original file line number Diff line number Diff line change
Expand Up @@ -469,9 +469,16 @@ export const useCallFlattenedTraceTree = (
childCallLookup
);
pathPrefix = updatePath(pathPrefix, currentCall.spanName, idx);
currentCall = currentCall.parentId
? traceCallMap[currentCall.parentId]
: null;
if (currentCall.parentId) {
if (!traceCallMap[currentCall.parentId]) {
// Cant find parent, assume it doesn't exist
currentCall.parentId = null;
} else {
currentCall = traceCallMap[currentCall.parentId];
}
} else {
currentCall = null;
}
}

// Add a parent row
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ export const CustomGridTreeDataGroupingCell: FC<
> = props => {
const {id, field, rowNode, row} = props;
const {isParentRow} = row;
const call = row.call as CallSchema;
const call = row.call as CallSchema | undefined;
const apiRef = useGridApiContext();
const handleClick: ButtonProps['onClick'] = event => {
if (rowNode.type !== 'group') {
Expand Down Expand Up @@ -90,6 +90,10 @@ export const CustomGridTreeDataGroupingCell: FC<

const isHiddenCount = id === 'HIDDEN_SIBLING_COUNT';

if (call == null) {
return <div />;
}

const box = (
<CursorBox
$isClickable={!isHiddenCount}
Expand Down Expand Up @@ -203,7 +207,7 @@ export const CustomGridTreeDataGroupingCell: FC<
{call.displayName ?? opNiceName(call.spanName)}
</Box>
</Box>
{call?.traceCall?.summary && (
{call.traceCall?.summary && (
<TraceCostStats
usageData={call.traceCall.summary.usage}
costData={call.traceCall.summary.weave?.costs}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import {Box} from '@mui/material';
import {Select} from '@wandb/weave/components/Form/Select';
import React from 'react';

import {LLM_MAX_TOKENS} from '../llmMaxTokens';

interface LLMDropdownProps {
value: string;
onChange: (value: string, maxTokens: number) => void;
}

export const LLMDropdown: React.FC<LLMDropdownProps> = ({value, onChange}) => {
const options = Object.keys(LLM_MAX_TOKENS).map(llm => ({
value: llm,
label: llm,
}));

return (
<Box
sx={{
width: '150px',
'& #react-select-2-listbox': {
width: '300px',
maxHeight: '500px',
},
'& #react-select-2-listbox > div': {
maxHeight: '500px',
},
}}>
<Select
value={options.find(opt => opt.value === value)}
onChange={option => {
if (option) {
const maxTokens =
LLM_MAX_TOKENS[
(option as {value: string}).value as keyof typeof LLM_MAX_TOKENS
]?.max_tokens || 0;
onChange((option as {value: string}).value, maxTokens);
}
}}
options={options}
size="medium"
isSearchable
/>
</Box>
);
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
import {Box, CircularProgress, Divider} from '@mui/material';
import {MOON_200} from '@wandb/weave/common/css/color.styles';
import React, {SetStateAction, useState} from 'react';

import {PlaygroundState, PlaygroundStateKey} from '../types';
import {PlaygroundChatTopBar} from './PlaygroundChatTopBar';

export type PlaygroundChatProps = {
entity: string;
project: string;
setPlaygroundStates: (states: PlaygroundState[]) => void;
playgroundStates: PlaygroundState[];
setPlaygroundStateField: (
index: number,
field: PlaygroundStateKey,
value: SetStateAction<PlaygroundState[PlaygroundStateKey]>
) => void;
setSettingsTab: (callIndex: number | null) => void;
settingsTab: number | null;
};

export const PlaygroundChat = ({
entity,
project,
setPlaygroundStates,
playgroundStates,
setPlaygroundStateField,
setSettingsTab,
settingsTab,
}: PlaygroundChatProps) => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const [isLoading, setIsLoading] = useState(false);
const chatPercentWidth = 100 / playgroundStates.length;

return (
<Box
sx={{
width: '100%',
height: '100%',
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
}}>
<Box
sx={{
width: '100%',
height: '100%',
maxHeight: 'calc(100% - 130px)',
display: 'flex',
position: 'relative',
}}>
{isLoading && (
<Box
sx={{
position: 'absolute',
top: 0,
left: 0,
right: 0,
bottom: 0,
backgroundColor: 'rgba(255, 255, 255, 0.7)',
zIndex: 100,
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
}}>
<CircularProgress />
</Box>
)}
{playgroundStates.map((state, idx) => (
<React.Fragment key={idx}>
{idx > 0 && (
<Divider
orientation="vertical"
flexItem
sx={{
height: '100%',
borderRight: `1px solid ${MOON_200}`,
}}
/>
)}
<Box
sx={{
width: '100%',
height: '100%',
display: 'flex',
flexDirection: 'column',
}}>
<Box
sx={{
position: 'absolute',
top: '8px',
left: `calc(${idx * chatPercentWidth}% + 8px)`,
width: `calc(${chatPercentWidth}% - 16px)`,
zIndex: 10,
}}>
<PlaygroundChatTopBar
idx={idx}
settingsTab={settingsTab}
setSettingsTab={setSettingsTab}
setPlaygroundStateField={setPlaygroundStateField}
setPlaygroundStates={setPlaygroundStates}
playgroundStates={playgroundStates}
entity={entity}
project={project}
/>
</Box>
</Box>
</React.Fragment>
))}
</Box>
</Box>
);
};
Loading

0 comments on commit fd0d7b7

Please sign in to comment.