Skip to content

Commit

Permalink
feat: add custom agents drop down
Browse files Browse the repository at this point in the history
  • Loading branch information
@louis030195
louis030195 committed Nov 4, 2024
1 parent 3c2010e commit 4393ce5
Show file tree
Hide file tree
Showing 4 changed files with 275 additions and 99 deletions.
197 changes: 138 additions & 59 deletions screenpipe-app-tauri/app/timeline/page.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,69 @@ interface TimeRange {
end: Date;
}

interface Agent {
id: string;
name: string;
description: string;
dataSelector: (frames: StreamTimeSeriesResponse[]) => any;
}

const AGENTS: Agent[] = [
{
id: "context-master",
name: "context master",
description: "analyzes everything: apps, windows, text & audio",
dataSelector: (frames) =>
frames.map((frame) => ({
timestamp: frame.timestamp,
devices: frame.devices.map((device) => ({
device_id: device.device_id,
metadata: device.metadata,
audio: device.audio,
})),
})),
},
{
id: "window-tracker",
name: "window tracker",
description: "focuses on app switching patterns",
dataSelector: (frames) =>
frames.map((frame) => ({
timestamp: frame.timestamp,
windows: frame.devices.map((device) => ({
app: device.metadata.app_name,
window: device.metadata.window_name,
})),
})),
},
{
id: "text-scanner",
name: "text scanner",
description: "analyzes visible text (OCR)",
dataSelector: (frames) =>
frames.map((frame) => ({
timestamp: frame.timestamp,
text: frame.devices
.map((device) => device.metadata.ocr_text)
.filter(Boolean),
})),
},
{
id: "voice-analyzer",
name: "voice analyzer",
description: "focuses on audio transcriptions",
dataSelector: (frames) =>
frames.map((frame) => ({
timestamp: frame.timestamp,
audio: frame.devices.flatMap((device) => device.audio),
})),
},
];

export default function Timeline() {
const [currentFrame, setCurrentFrame] = useState<DeviceFrameResponse | null>(null);
const [currentFrame, setCurrentFrame] = useState<DeviceFrameResponse | null>(
null
);
const [frames, setFrames] = useState<StreamTimeSeriesResponse[]>([]);
const [currentIndex, setCurrentIndex] = useState(0);
const [isLoading, setIsLoading] = useState(true);
Expand Down Expand Up @@ -83,6 +144,7 @@ export default function Timeline() {
const [isDraggingPanel, setIsDraggingPanel] = useState(false);
const [dragOffset, setDragOffset] = useState({ x: 0, y: 0 });
const [osType, setOsType] = useState<string>("");
const [selectedAgent, setSelectedAgent] = useState<Agent>(AGENTS[0]);

useEffect(() => {
setPosition({
Expand Down Expand Up @@ -121,12 +183,15 @@ export default function Timeline() {
setFrames((prev) => {
const exists = prev.some((f) => f.timestamp === data.timestamp);
if (exists) return prev;

// ! HACK: Add new frame and sort in descending order
const newFrames = [...prev, data].sort((a, b) => {
return new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime();
return (
new Date(b.timestamp).getTime() -
new Date(a.timestamp).getTime()
);
});

return newFrames;
});

Expand Down Expand Up @@ -159,18 +224,20 @@ export default function Timeline() {
};

const getLoadedTimeRangeStyles = () => {
if (!loadedTimeRange || frames.length === 0) return { left: "0%", right: "100%" };
if (!loadedTimeRange || frames.length === 0)
return { left: "0%", right: "100%" };

// Find the earliest frame timestamp
const firstFrame = [...frames].sort((a, b) =>
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
const firstFrame = [...frames].sort(
(a, b) =>
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
)[0];
const earliestTime = new Date(firstFrame.timestamp);

// Create local date objects for today
const startOfDay = new Date();
startOfDay.setHours(0, 0, 0, 0);

const endOfDay = new Date();
endOfDay.setHours(23, 59, 59, 999);

Expand All @@ -179,8 +246,10 @@ export default function Timeline() {
const localEnd = new Date(loadedTimeRange.end);

const totalMs = endOfDay.getTime() - startOfDay.getTime();
const startPercent = ((localStart.getTime() - startOfDay.getTime()) / totalMs) * 100;
const endPercent = ((localEnd.getTime() - startOfDay.getTime()) / totalMs) * 100;
const startPercent =
((localStart.getTime() - startOfDay.getTime()) / totalMs) * 100;
const endPercent =
((localEnd.getTime() - startOfDay.getTime()) / totalMs) * 100;

// Temporarily return empty values to disable grey areas
return {
Expand Down Expand Up @@ -261,8 +330,10 @@ export default function Timeline() {

const getCurrentTimePercentage = () => {
if (!currentFrame) return 0;

const frameTime = new Date(currentFrame.metadata.timestamp || frames[currentIndex].timestamp);

const frameTime = new Date(
currentFrame.metadata.timestamp || frames[currentIndex].timestamp
);
const startOfDay = new Date(frameTime);
startOfDay.setHours(0, 0, 0, 0);
const endOfDay = new Date(frameTime);
Expand All @@ -278,10 +349,10 @@ export default function Timeline() {
const rect = e.currentTarget.getBoundingClientRect();
const clickX = e.clientX - rect.left;
const percentage = (clickX / rect.width) * 100;

// Clamp percentage between 0 and 100
const clampedPercentage = Math.max(0, Math.min(100, percentage));

setIsDragging(true);
setDragStart(clampedPercentage);

Expand Down Expand Up @@ -317,13 +388,15 @@ export default function Timeline() {
const rect = e.currentTarget.getBoundingClientRect();
const moveX = e.clientX - rect.left;
const percentage = (moveX / rect.width) * 100;

// Clamp percentage between 0 and 100
const clampedPercentage = Math.max(0, Math.min(100, percentage));

const totalMinutesInDay = 24 * 60;
const startMinutes = (Math.min(dragStart, clampedPercentage) / 100) * totalMinutesInDay;
const endMinutes = (Math.max(dragStart, clampedPercentage) / 100) * totalMinutesInDay;
const startMinutes =
(Math.min(dragStart, clampedPercentage) / 100) * totalMinutesInDay;
const endMinutes =
(Math.max(dragStart, clampedPercentage) / 100) * totalMinutesInDay;

const startLocal = new Date();
startLocal.setHours(
Expand Down Expand Up @@ -393,16 +466,14 @@ export default function Timeline() {
e.preventDefault();
if (!aiInput.trim() || !selectionRange) return;

console.log("selectionRange", selectionRange);
console.log("frames", frames);
const relevantFrames = frames.filter((frame) => {
const frameTime = new Date(frame.timestamp);
console.log("frameTime", frameTime);
return (
frameTime >= selectionRange.start && frameTime <= selectionRange.end
);
});
console.log("relevantFrames", relevantFrames);

const contextData = selectedAgent.dataSelector(relevantFrames);

const userMessage = {
id: generateId(),
Expand All @@ -423,30 +494,23 @@ export default function Timeline() {
const messages = [
{
role: "system" as const,
content: `You are a helpful assistant analyzing screen & mic 24/7 recordings including OCR, transcription, and more.
content: `You are a helpful assistant specialized as a "${
selectedAgent.name
}" analyzing screen & mic recordings.
Rules:
- Current time (JavaScript Date.prototype.toString): ${new Date().toString()}
- User timezone: ${Intl.DateTimeFormat().resolvedOptions().timeZone}
- User timezone offset: ${new Date().getTimezoneOffset()}
- All timestamps in the context data are in UTC
- Convert timestamps to local time for human-readable responses
- Never output UTC time unless explicitly asked
- Focus on app usage patterns and context switches
- Focus on ${selectedAgent.description}
`,
},
...chatMessages,
{
role: "user" as const,
content: `Context data: ${JSON.stringify(
relevantFrames.map((frame) => ({
timestamp: frame.timestamp,
devices: frame.devices.map(device => ({
device_id: device.device_id,
metadata: device.metadata,
audio: device.audio,
}))
}))
)}
content: `Context data: ${JSON.stringify(contextData)}
${aiInput}`,
},
Expand Down Expand Up @@ -620,7 +684,9 @@ export default function Timeline() {
<div className="flex items-center gap-4">
<div>device: {currentFrame.device_id}</div>
<div>app: {currentFrame.metadata.app_name || "n/a"}</div>
<div>window: {currentFrame.metadata.window_name || "n/a"}</div>
<div>
window: {currentFrame.metadata.window_name || "n/a"}
</div>
</div>
</div>
</div>
Expand Down Expand Up @@ -784,32 +850,45 @@ export default function Timeline() {
msUserSelect: "text",
}}
>
<div className="flex gap-2">
<Input
ref={inputRef}
type="text"
value={aiInput}
onChange={(e) => setAiInput(e.target.value)}
placeholder="ask about this time range..."
className="flex-1 bg-black/50 border-[#0f0] text-[#0f0] placeholder-[#0f0]/50"
disabled={isAiLoading}
/>
<Button
type="submit"
disabled={isAiLoading}
className="bg-[#0f0]/20 hover:bg-[#0f0]/30 border border-[#0f0] text-[#0f0] group relative"
<div className="flex flex-col gap-2">
<select
value={selectedAgent.id}
onChange={(e) => setSelectedAgent(AGENTS.find(a => a.id === e.target.value) || AGENTS[0])}
className="w-full bg-black/50 border border-[#0f0] text-[#0f0] rounded px-2 py-1 text-xs"
>
{isStreaming ? (
<Square className="h-4 w-4" />
) : (
<>
<Send className="h-4 w-4" />
<span className="absolute -top-8 right-0 text-[10px] opacity-0 group-hover:opacity-100 transition-opacity bg-black/90 px-2 py-1 rounded whitespace-nowrap">
{osType === "macos" ? "⌘↵" : "Ctrl+↵"}
</span>
</>
)}
</Button>
{AGENTS.map((agent) => (
<option key={agent.id} value={agent.id} className="bg-black">
{agent.name} - {agent.description}
</option>
))}
</select>
<div className="flex gap-2">
<Input
ref={inputRef}
type="text"
value={aiInput}
onChange={(e) => setAiInput(e.target.value)}
placeholder="ask about this time range..."
className="flex-1 bg-black/50 border-[#0f0] text-[#0f0] placeholder-[#0f0]/50"
disabled={isAiLoading}
/>
<Button
type="submit"
disabled={isAiLoading}
className="bg-[#0f0]/20 hover:bg-[#0f0]/30 border border-[#0f0] text-[#0f0] group relative"
>
{isStreaming ? (
<Square className="h-4 w-4" />
) : (
<>
<Send className="h-4 w-4" />
<span className="absolute -top-8 right-0 text-[10px] opacity-0 group-hover:opacity-100 transition-opacity bg-black/90 px-2 py-1 rounded whitespace-nowrap">
{osType === "macos" ? "⌘↵" : "Ctrl+↵"}
</span>
</>
)}
</Button>
</div>
</div>
</form>
</div>
Expand Down
Loading

0 comments on commit 4393ce5

Please sign in to comment.