From aae824cb9bb9920856417b7157ef10c08f39999e Mon Sep 17 00:00:00 2001 From: Brendan Allan Date: Tue, 26 Nov 2024 23:06:00 +0800 Subject: [PATCH] Support multiple segments in recording meta (#177) * support multiple segments for video * cargo fmt * multi-segment metadata w/o cursors * clippy * multi-segment cursor recording * fix types * update ts * put back zoom segments * extra tests --- apps/desktop/package.json | 6 +- apps/desktop/src-tauri/src/auth.rs | 1 - apps/desktop/src-tauri/src/export.rs | 3 +- apps/desktop/src-tauri/src/lib.rs | 21 +- .../src-tauri/src/platform/macos/mod.rs | 3 - apps/desktop/src-tauri/src/recording.rs | 169 +++++--- apps/desktop/src/App.tsx | 18 +- .../src/routes/editor/ConfigSidebar.tsx | 4 +- apps/desktop/src/routes/editor/Editor.tsx | 26 +- apps/desktop/src/routes/editor/Player.tsx | 2 +- apps/desktop/src/routes/editor/Timeline.tsx | 22 +- apps/desktop/src/utils/tauri.ts | 11 +- apps/embed/package.json | 2 +- apps/storybook/package.json | 2 +- apps/tasks/package.json | 16 +- apps/web/package.json | 2 +- crates/editor/src/editor_instance.rs | 115 ++++-- crates/editor/src/lib.rs | 4 +- crates/editor/src/playback.rs | 37 +- crates/editor/src/project_recordings.rs | 76 +++- crates/export/src/lib.rs | 98 +++-- crates/flags/src/lib.rs | 8 +- crates/media/src/feeds/audio.rs | 4 +- crates/media/src/sources/screen_capture.rs | 2 +- crates/project/src/configuration.rs | 7 +- crates/project/src/cursor.rs | 29 +- crates/project/src/lib.rs | 140 +------ crates/project/src/meta.rs | 344 ++++++++++++++++ crates/recording/src/actor.rs | 141 +++---- crates/recording/src/cursor.rs | 227 +++++----- crates/recording/src/lib.rs | 6 +- crates/recording/src/segmented_actor.rs | 244 +++++++++-- crates/rendering/src/decoder.rs | 46 ++- crates/rendering/src/lib.rs | 138 +++---- packages/database/package.json | 2 +- packages/ui-solid/src/auto-imports.d.ts | 2 + packages/ui/package.json | 2 +- packages/utils/package.json | 2 +- pnpm-lock.yaml | 387 +++++++++--------- 39 files changed, 1488 insertions(+), 881 deletions(-) create mode 100644 crates/project/src/meta.rs diff --git a/apps/desktop/package.json b/apps/desktop/package.json index ede86a6b..37eb85b1 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -3,7 +3,7 @@ "type": "module", "scripts": { "dev": "dotenv -e ../../.env -- tauri dev", - "localdev": "vinxi dev --port 3001", + "localdev": "dotenv -e ../../.env -- vinxi dev --port 3001", "build": "vinxi build", "tauri": "tauri" }, @@ -34,12 +34,12 @@ "@tauri-apps/plugin-dialog": "2.0.0-rc.1", "@tauri-apps/plugin-fs": "2.0.0-rc.0", "@tauri-apps/plugin-http": "^2.0.1", + "@tauri-apps/plugin-notification": "2.0.0-rc.0", "@tauri-apps/plugin-os": "2.0.0-rc.1", "@tauri-apps/plugin-process": "2.0.0-rc.0", "@tauri-apps/plugin-shell": ">=2.0.0-rc.0", "@tauri-apps/plugin-store": "2.1.0", "@tauri-apps/plugin-updater": "2.0.0-rc.0", - "@tauri-apps/plugin-notification": "2.0.0-rc.0", "@types/react-tooltip": "^4.2.4", "cva": "npm:class-variance-authority@^0.7.0", "effect": "^3.7.2", @@ -59,7 +59,7 @@ "@iconify/json": "^2.2.239", "@tauri-apps/cli": ">=2.0.0-rc.0", "@types/dom-webcodecs": "^0.1.11", - "typescript": "^5.2.2", + "typescript": "^5.7.2", "vite": "^5.4.3", "vite-tsconfig-paths": "^5.0.1" } diff --git a/apps/desktop/src-tauri/src/auth.rs b/apps/desktop/src-tauri/src/auth.rs index 8fa53681..44d26d88 100644 --- a/apps/desktop/src-tauri/src/auth.rs +++ b/apps/desktop/src-tauri/src/auth.rs @@ -1,4 +1,3 @@ - use serde::{Deserialize, Serialize}; use serde_json::json; use specta::Type; diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index cf9d481f..615031e1 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -48,10 +48,9 @@ pub async fn export_video( .ok(); }, &editor_instance.project_path, - editor_instance.audio.clone(), editor_instance.meta(), editor_instance.render_constants.clone(), - editor_instance.cursor.clone(), + &editor_instance.segments, ) .await .map_err(|e| { diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 6e43158e..f3eaf78b 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -19,8 +19,8 @@ mod windows; use audio::AppSounds; use auth::{AuthStore, AuthenticationInvalid}; -use cap_editor::{EditorInstance, FRAMES_WS_PATH}; -use cap_editor::{EditorState, ProjectRecordings}; +use cap_editor::EditorState; +use cap_editor::{EditorInstance, ProjectRecordings, FRAMES_WS_PATH}; use cap_media::feeds::{AudioInputFeed, AudioInputSamplesSender}; use cap_media::sources::CaptureScreen; use cap_media::{ @@ -751,7 +751,7 @@ async fn create_editor_instance( let project_config = editor_instance.project_config.1.borrow(); project_config.clone() }, - recordings: editor_instance.recordings, + recordings: editor_instance.recordings.clone(), path: editor_instance.project_path.clone(), pretty_name: meta.pretty_name, }) @@ -1320,13 +1320,16 @@ async fn take_screenshot(app: AppHandle, _state: MutableState<'_, App>) -> Resul project_path: recording_dir.clone(), sharing: None, pretty_name: screenshot_name, - display: Display { - path: screenshot_path.clone(), + content: cap_project::Content::SingleSegment { + segment: cap_project::SingleSegment { + display: Display { + path: screenshot_path.clone(), + }, + camera: None, + audio: None, + cursor: None, + }, }, - camera: None, - audio: None, - segments: vec![], - cursor: None, } .save_for_project(); diff --git a/apps/desktop/src-tauri/src/platform/macos/mod.rs b/apps/desktop/src-tauri/src/platform/macos/mod.rs index 058984b6..c9444131 100644 --- a/apps/desktop/src-tauri/src/platform/macos/mod.rs +++ b/apps/desktop/src-tauri/src/platform/macos/mod.rs @@ -12,9 +12,6 @@ use objc::{class, msg_send, sel, sel_impl}; pub mod delegates; -use specta::Type; -use tauri_specta::Event; - #[derive(Debug)] pub struct Window { pub window_number: u32, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index bda7fe90..c4a4a453 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -1,3 +1,5 @@ +use std::time::Instant; + use crate::{ audio::AppSounds, auth::AuthStore, @@ -12,11 +14,14 @@ use crate::{ RecordingStarted, RecordingStopped, UploadMode, }; use cap_editor::ProjectRecordings; +use cap_flags::FLAGS; use cap_media::feeds::CameraFeed; use cap_media::sources::{AVFrameCapture, CaptureScreen, CaptureWindow, ScreenCaptureSource}; -use cap_project::{ProjectConfiguration, TimelineConfiguration, TimelineSegment, ZoomSegment}; +use cap_project::{ + Content, ProjectConfiguration, TimelineConfiguration, TimelineSegment, ZoomSegment, +}; +use cap_recording::CompletedRecording; use cap_rendering::ZOOM_DURATION; -use std::time::Instant; use tauri::{AppHandle, Manager}; use tauri_specta::Event; @@ -120,22 +125,24 @@ pub async fn start_recording(app: AppHandle, state: MutableState<'_, App>) -> Re #[tauri::command] #[specta::specta] pub async fn pause_recording(state: MutableState<'_, App>) -> Result<(), String> { - let state = state.write().await; + let mut state = state.write().await; + + if let Some(recording) = state.current_recording.as_mut() { + recording.pause().await.map_err(|e| e.to_string())?; + } - // if let Some(recording) = &mut state.current_recording { - // recording.pause().await.map_err(|e| e.to_string())?; - // } Ok(()) } #[tauri::command] #[specta::specta] pub async fn resume_recording(state: MutableState<'_, App>) -> Result<(), String> { - let state = state.write().await; + let mut state = state.write().await; + + if let Some(recording) = state.current_recording.as_mut() { + recording.resume().await.map_err(|e| e.to_string())?; + } - // if let Some(recording) = &mut state.current_recording { - // recording.resume().await.map_err(|e| e.to_string())?; - // } Ok(()) } @@ -152,7 +159,7 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res }; let now = Instant::now(); - let recording = current_recording.stop().await.map_err(|e| e.to_string())?; + let completed_recording = current_recording.stop().await.map_err(|e| e.to_string())?; println!("stopped recording in {:?}", now.elapsed()); if let Some(window) = CapWindowId::InProgressRecording.get(&app) { @@ -163,17 +170,21 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res window.unminimize().ok(); } - let screenshots_dir = recording.recording_dir.join("screenshots"); + let screenshots_dir = completed_recording.recording_dir.join("screenshots"); std::fs::create_dir_all(&screenshots_dir).ok(); + let display_output_path = match &completed_recording.meta.content { + Content::SingleSegment { segment } => { + segment.path(&completed_recording.meta, &segment.display.path) + } + Content::MultipleSegments { inner } => { + inner.path(&completed_recording.meta, &inner.segments[0].display.path) + } + }; + let display_screenshot = screenshots_dir.join("display.jpg"); let now = Instant::now(); - create_screenshot( - recording.display_output_path.clone(), - display_screenshot.clone(), - None, - ) - .await?; + create_screenshot(display_output_path, display_screenshot.clone(), None).await?; println!("created screenshot in {:?}", now.elapsed()); // let thumbnail = screenshots_dir.join("thumbnail.png"); @@ -181,7 +192,7 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res // create_thumbnail(display_screenshot, thumbnail, (100, 100)).await?; // println!("created thumbnail in {:?}", now.elapsed()); - let recording_dir = recording.recording_dir.clone(); + let recording_dir = completed_recording.recording_dir.clone(); ShowCapWindow::PrevRecordings.show(&app).ok(); @@ -197,60 +208,37 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res .emit(&app) .ok(); - let recordings = ProjectRecordings::new(&recording.meta); - let max_duration = recordings.duration(); + let recordings = ProjectRecordings::new(&completed_recording.meta); let config = { let segments = { let mut segments = vec![]; let mut passed_duration = 0.0; - for i in (0..recording.segments.len()).step_by(2) { + // multi-segment + // for segment in &completed_recording.segments { + // let start = passed_duration; + // passed_duration += segment.end - segment.start; + // segments.push(TimelineSegment { + // recording_segment: None, + // start, + // end: passed_duration.min(recordings.duration()), + // timescale: 1.0, + // }); + // } + + // single-segment + for i in (0..completed_recording.segments.len()).step_by(2) { let start = passed_duration; - passed_duration += recording.segments[i + 1] - recording.segments[i]; + passed_duration += + completed_recording.segments[i + 1] - completed_recording.segments[i]; segments.push(TimelineSegment { + recording_segment: None, start, end: passed_duration.min(recordings.duration()), timescale: 1.0, }); } - segments - }; - - let zoom_segments = { - let mut segments = vec![]; - - const ZOOM_SEGMENT_AFTER_CLICK_PADDING: f64 = 1.5; - - for click in &recording.cursor_data.clicks { - let time = click.process_time_ms / 1000.0; - - if segments.last().is_none() { - segments.push(ZoomSegment { - start: (click.process_time_ms / 1000.0 - (ZOOM_DURATION + 0.2)).max(0.0), - end: click.process_time_ms / 1000.0 + ZOOM_SEGMENT_AFTER_CLICK_PADDING, - amount: 2.0, - }); - } else { - let last_segment = segments.last_mut().unwrap(); - - if click.down { - if last_segment.end > time { - last_segment.end = (time + ZOOM_SEGMENT_AFTER_CLICK_PADDING) - .min(recordings.duration()); - } else if time < max_duration - ZOOM_DURATION { - segments.push(ZoomSegment { - start: (time - ZOOM_DURATION).max(0.0), - end: time + ZOOM_SEGMENT_AFTER_CLICK_PADDING, - amount: 2.0, - }); - } - } else { - last_segment.end = - (time + ZOOM_SEGMENT_AFTER_CLICK_PADDING).min(recordings.duration()); - } - } - } segments }; @@ -258,14 +246,17 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res ProjectConfiguration { timeline: Some(TimelineConfiguration { segments, - zoom_segments, + zoom_segments: generate_zoom_segments_from_clicks( + &completed_recording, + &recordings, + ), }), ..Default::default() } }; config - .write(&recording.recording_dir) + .write(&completed_recording.recording_dir) .map_err(|e| e.to_string())?; AppSounds::StopRecording.play(); @@ -299,7 +290,7 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res export_video( app.clone(), - recording.id.clone(), + completed_recording.id.clone(), config, tauri::ipc::Channel::new(|_| Ok(())), true, @@ -310,7 +301,7 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res while retry_count < max_retries { match upload_exported_video( app.clone(), - recording.id.clone(), + completed_recording.id.clone(), UploadMode::Initial { pre_created_video: Some(pre_created_video.clone()), }, @@ -344,7 +335,7 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res }); } } else if settings.open_editor_after_recording { - open_editor(app.clone(), recording.id); + open_editor(app.clone(), completed_recording.id); } } @@ -352,3 +343,51 @@ pub async fn stop_recording(app: AppHandle, state: MutableState<'_, App>) -> Res Ok(()) } + +fn generate_zoom_segments_from_clicks( + recording: &CompletedRecording, + recordings: &ProjectRecordings, +) -> Vec { + let mut segments = vec![]; + + if !FLAGS.zoom { + return vec![]; + }; + + let max_duration = recordings.duration(); + + const ZOOM_SEGMENT_AFTER_CLICK_PADDING: f64 = 1.5; + + // single-segment only + for click in &recording.cursor_data.clicks { + let time = click.process_time_ms / 1000.0; + + if segments.last().is_none() { + segments.push(ZoomSegment { + start: (click.process_time_ms / 1000.0 - (ZOOM_DURATION + 0.2)).max(0.0), + end: click.process_time_ms / 1000.0 + ZOOM_SEGMENT_AFTER_CLICK_PADDING, + amount: 2.0, + }); + } else { + let last_segment = segments.last_mut().unwrap(); + + if click.down { + if last_segment.end > time { + last_segment.end = + (time + ZOOM_SEGMENT_AFTER_CLICK_PADDING).min(recordings.duration()); + } else if time < max_duration - ZOOM_DURATION { + segments.push(ZoomSegment { + start: (time - ZOOM_DURATION).max(0.0), + end: time + ZOOM_SEGMENT_AFTER_CLICK_PADDING, + amount: 2.0, + }); + } + } else { + last_segment.end = + (time + ZOOM_SEGMENT_AFTER_CLICK_PADDING).min(recordings.duration()); + } + } + } + + segments +} diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index d2c4976d..153bebe8 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -48,20 +48,6 @@ const queryClient = new QueryClient({ export default function App() { const darkMode = themeStore.isDarkMode; - onMount(async () => { - await themeStore.initialize(); - - const matches = useCurrentMatches(); - - onMount(() => { - for (const match of matches()) { - if (match.route.info?.AUTO_SHOW_WINDOW === false) return; - } - - getCurrentWindow().show(); - }); - }); - return (
{ + getCurrentWindow().show(); + }); }); return {props.children}; diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index 70b6091b..a5e2475d 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -91,7 +91,9 @@ export function ConfigSidebar() { { id: "camera" as const, icon: IconCapCamera, - disabled: editorInstance.recordings.camera === null, + disabled: editorInstance.recordings.segments.every( + (s) => s.camera === null + ), }, // { // id: "transcript" as const, diff --git a/apps/desktop/src/routes/editor/Editor.tsx b/apps/desktop/src/routes/editor/Editor.tsx index 7e209c6d..92635f6c 100644 --- a/apps/desktop/src/routes/editor/Editor.tsx +++ b/apps/desktop/src/routes/editor/Editor.tsx @@ -318,7 +318,7 @@ function Dialogs() { size: dialog().size, }); - const display = editorInstance.recordings.display; + const display = editorInstance.recordings.segments[0].display; const styles = createMemo(() => { return { @@ -377,8 +377,8 @@ function Dialogs() { setCrop({ position: { x: 0, y: 0 }, size: { - x: editorInstance.recordings.display.width, - y: editorInstance.recordings.display.height, + x: display.width, + y: display.height, }, }) } @@ -531,8 +531,7 @@ function Dialogs() { clamp( original.size.x + diff.x, MIN_SIZE, - editorInstance.recordings - .display.width - + display.width - crop.position.x ) ) @@ -546,9 +545,7 @@ function Dialogs() { original.position.x + diff.x, 0, - editorInstance.recordings - .display.width - - MIN_SIZE + display.width - MIN_SIZE ) ) ); @@ -559,8 +556,7 @@ function Dialogs() { clamp( original.size.x - diff.x, MIN_SIZE, - editorInstance.recordings - .display.width + display.width ) ) ); @@ -574,8 +570,7 @@ function Dialogs() { clamp( original.size.y + diff.y, MIN_SIZE, - editorInstance.recordings - .display.height - + display.height - crop.position.y ) ) @@ -589,9 +584,7 @@ function Dialogs() { original.position.y + diff.y, 0, - editorInstance.recordings - .display.height - - MIN_SIZE + display.height - MIN_SIZE ) ) ); @@ -602,8 +595,7 @@ function Dialogs() { clamp( original.size.y - diff.y, MIN_SIZE, - editorInstance.recordings - .display.height + display.height ) ) ); diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index a37f1e57..348835ee 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -118,7 +118,7 @@ export function Player() { } onClick={() => { - const display = editorInstance.recordings.display; + const display = editorInstance.recordings.segments[0].display; setDialog({ open: true, type: "crop", diff --git a/apps/desktop/src/routes/editor/Timeline.tsx b/apps/desktop/src/routes/editor/Timeline.tsx index 2bddee74..b80fbd38 100644 --- a/apps/desktop/src/routes/editor/Timeline.tsx +++ b/apps/desktop/src/routes/editor/Timeline.tsx @@ -18,7 +18,7 @@ import { mergeRefs } from "@solid-primitives/refs"; import { createContextProvider } from "@solid-primitives/context"; import { createMemo } from "solid-js"; -import { commands } from "~/utils/tauri"; +import { commands, TimelineSegment } from "~/utils/tauri"; import { useEditorContext } from "./context"; import { formatTime } from "./utils"; @@ -54,7 +54,9 @@ export function Timeline() { if (!project.timeline) { const resume = history.pause(); setProject("timeline", { - segments: [{ timescale: 1, start: 0, end: duration() }], + segments: [ + { timescale: 1, start: 0, end: duration(), recordingSegment: null }, + ], }); resume(); } @@ -62,8 +64,10 @@ export function Timeline() { const xPadding = 12; - const segments = () => - project.timeline?.segments ?? [{ start: 0, end: duration(), timescale: 1 }]; + const segments = (): Array => + project.timeline?.segments ?? [ + { start: 0, end: duration(), timescale: 1, recordingSegment: null }, + ]; if (window.FLAGS.zoom) if ( @@ -73,7 +77,14 @@ export function Timeline() { setProject( produce((project) => { project.timeline ??= { - segments: [{ start: 0, end: duration(), timescale: 1 }], + segments: [ + { + start: 0, + end: duration(), + timescale: 1, + recordingSegment: null, + }, + ], }; }) ); @@ -176,6 +187,7 @@ export function Timeline() { start: splitTime, end: segment.end, timescale: 1, + recordingSegment: segment.recordingSegment, }); segments[i()].end = splitTime; }) diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index b01b364b..75fac8e7 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -240,6 +240,8 @@ export type HotkeyAction = "startRecording" | "stopRecording" | "restartRecordin export type HotkeysConfiguration = { show: boolean } export type HotkeysStore = { hotkeys: { [key in HotkeyAction]: Hotkey } } export type JsonValue = [T] +export type MultipleSegment = { display: Display; camera?: CameraMeta | null; audio?: AudioMeta | null; cursor?: string | null } +export type MultipleSegments = { segments: MultipleSegment[]; cursors: { [key in string]: string } } export type NewNotification = { title: string; body: string; is_error: boolean } export type NewRecordingAdded = { path: string } export type NewScreenshotAdded = { path: string } @@ -249,13 +251,12 @@ export type OSPermissionsCheck = { screenRecording: OSPermissionStatus; micropho export type Plan = { upgraded: boolean; last_checked: number } export type PreCreatedVideo = { id: string; link: string; config: S3UploadMeta } export type ProjectConfiguration = { aspectRatio: AspectRatio | null; background: BackgroundConfiguration; camera: Camera; audio: AudioConfiguration; cursor: CursorConfiguration; hotkeys: HotkeysConfiguration; timeline?: TimelineConfiguration | null; motionBlur: number | null } -export type ProjectRecordings = { display: Video; camera: Video | null; audio: Audio | null } +export type ProjectRecordings = { segments: SegmentRecordings[] } export type RecordingInfo = { captureTarget: ScreenCaptureTarget } -export type RecordingMeta = { pretty_name: string; sharing?: SharingMeta | null; display: Display; camera?: CameraMeta | null; audio?: AudioMeta | null; segments?: RecordingSegment[]; cursor: string | null } +export type RecordingMeta = ({ segment: SingleSegment } | { inner: MultipleSegments }) & { pretty_name: string; sharing?: SharingMeta | null } export type RecordingMetaChanged = { id: string } export type RecordingOptions = { captureTarget: ScreenCaptureTarget; cameraLabel: string | null; audioInputName: string | null } export type RecordingOptionsChanged = null -export type RecordingSegment = { start: number; end: number } export type RecordingStarted = null export type RecordingStopped = { path: string } export type RenderFrameEvent = { frame_number: number } @@ -267,11 +268,13 @@ export type RequestStartRecording = null export type RequestStopRecording = null export type S3UploadMeta = { id: string; user_id: string; aws_region?: string; aws_bucket?: string } export type ScreenCaptureTarget = ({ variant: "window" } & CaptureWindow) | ({ variant: "screen" } & CaptureScreen) +export type SegmentRecordings = { display: Video; camera: Video | null; audio: Audio | null } export type SerializedEditorInstance = { framesSocketUrl: string; recordingDuration: number; savedProjectConfig: ProjectConfiguration; recordings: ProjectRecordings; path: string; prettyName: string } export type SharingMeta = { id: string; link: string } export type ShowCapWindow = "Setup" | "Main" | { Settings: { page: string | null } } | { Editor: { project_id: string } } | "PrevRecordings" | "WindowCaptureOccluder" | { Camera: { ws_port: number } } | { InProgressRecording: { position: [number, number] | null } } | "Upgrade" +export type SingleSegment = { display: Display; camera?: CameraMeta | null; audio?: AudioMeta | null; cursor?: string | null } export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments?: ZoomSegment[] } -export type TimelineSegment = { timescale: number; start: number; end: number } +export type TimelineSegment = { recordingSegment: number | null; timescale: number; start: number; end: number } export type UploadMode = { Initial: { pre_created_video: PreCreatedVideo | null } } | "Reupload" export type UploadProgress = { stage: string; progress: number; message: string } export type UploadResult = { Success: string } | "NotAuthenticated" | "PlanCheckFailed" | "UpgradeRequired" diff --git a/apps/embed/package.json b/apps/embed/package.json index 7370a822..2905fdb5 100644 --- a/apps/embed/package.json +++ b/apps/embed/package.json @@ -43,7 +43,7 @@ "eslint-config-next": "14.1.0", "postcss": "^8.4.23", "tailwindcss": "^3", - "typescript": "^5.3.2" + "typescript": "^5.7.2" }, "engines": { "node": "20" diff --git a/apps/storybook/package.json b/apps/storybook/package.json index db866818..5fede3bf 100644 --- a/apps/storybook/package.json +++ b/apps/storybook/package.json @@ -22,7 +22,7 @@ "storybook": "^8.2.7", "storybook-solidjs": "^1.0.0-beta.2", "storybook-solidjs-vite": "^1.0.0-beta.2", - "typescript": "^5.2.2", + "typescript": "^5.7.2", "vite": "^5.3.4", "vite-plugin-solid": "^2.10.2" } diff --git a/apps/tasks/package.json b/apps/tasks/package.json index d3386381..ef7316e5 100644 --- a/apps/tasks/package.json +++ b/apps/tasks/package.json @@ -13,6 +13,13 @@ "typecheck": "tsc --noEmit" }, "dependencies": { + "@types/cors": "^2.8.17", + "@types/express": "^4.17.21", + "@types/fluent-ffmpeg": "^2.1.24", + "@types/jest": "^29.5.12", + "@types/morgan": "^1.9.9", + "@types/node": "^20.12.6", + "@types/supertest": "^6.0.2", "cors": "^2.8.5", "dotenv": "^16.4.5", "express": "^4.19.2", @@ -20,14 +27,7 @@ "helmet": "^7.1.0", "morgan": "^1.10.0", "ts-node": "^10.9.2", - "typescript": "^5.4.4", - "@types/cors": "^2.8.17", - "@types/express": "^4.17.21", - "@types/fluent-ffmpeg": "^2.1.24", - "@types/jest": "^29.5.12", - "@types/morgan": "^1.9.9", - "@types/node": "^20.12.6", - "@types/supertest": "^6.0.2" + "typescript": "^5.7.2" }, "devDependencies": { "@typescript-eslint/eslint-plugin": "^7.6.0", diff --git a/apps/web/package.json b/apps/web/package.json index 82d70c16..14a42e91 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -78,7 +78,7 @@ "eslint-config-next": "14.1.0", "postcss": "^8.4.23", "tailwindcss": "^3", - "typescript": "^5.3.2" + "typescript": "^5.7.2" }, "engines": { "node": "20" diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index 7776790e..f0eec37d 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -2,8 +2,11 @@ use crate::editor; use crate::playback::{self, PlaybackHandle}; use crate::project_recordings::ProjectRecordings; use cap_media::feeds::AudioData; -use cap_project::{CursorData, ProjectConfiguration, RecordingMeta, XY}; -use cap_rendering::{ProjectUniforms, RecordingDecoders, RenderOptions, RenderVideoConstants}; +use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, XY}; +use cap_rendering::{ + ProjectUniforms, RecordingSegmentDecoders, RenderOptions, RenderVideoConstants, + SegmentVideoPaths, +}; use std::ops::Deref; use std::sync::Mutex as StdMutex; use std::{path::PathBuf, sync::Arc}; @@ -14,10 +17,7 @@ const FPS: u32 = 30; pub struct EditorInstance { pub project_path: PathBuf, pub id: String, - pub audio: Arc>, - pub cursor: Arc, pub ws_port: u16, - pub decoders: RecordingDecoders, pub recordings: ProjectRecordings, pub renderer: Arc, pub render_constants: Arc, @@ -29,6 +29,7 @@ pub struct EditorInstance { watch::Receiver, ), ws_shutdown: Arc>>>, + pub segments: Arc>, } impl EditorInstance { @@ -62,33 +63,77 @@ impl EditorInstance { let recordings = ProjectRecordings::new(&meta); let render_options = RenderOptions { - screen_size: XY::new(recordings.display.width, recordings.display.height), - camera_size: recordings + screen_size: XY::new( + recordings.segments[0].display.width, + recordings.segments[0].display.height, + ), + camera_size: recordings.segments[0] .camera .as_ref() .map(|c| XY::new(c.width, c.height)), }; - let audio = meta.audio.as_ref().map(|meta| { - let audio_path = project_path.join(&meta.path); + let segments = + match &meta.content { + cap_project::Content::SingleSegment { segment: s } => { + let audio = + Arc::new(s.audio.as_ref().map(|meta| { + AudioData::from_file(project_path.join(&meta.path)).unwrap() + })); + + let cursor = Arc::new(s.cursor_data(&meta).into()); + + let decoders = RecordingSegmentDecoders::new( + &meta, + SegmentVideoPaths { + display: s.display.path.as_path(), + camera: s.camera.as_ref().map(|c| c.path.as_path()), + }, + ); + + vec![Segment { + audio, + cursor, + decoders, + }] + } + cap_project::Content::MultipleSegments { inner } => { + let mut segments = vec![]; + + for s in &inner.segments { + let audio = Arc::new(s.audio.as_ref().map(|meta| { + AudioData::from_file(project_path.join(&meta.path)).unwrap() + })); + + let cursor = Arc::new(s.cursor_events(&meta)); + + let decoders = RecordingSegmentDecoders::new( + &meta, + SegmentVideoPaths { + display: s.display.path.as_path(), + camera: s.camera.as_ref().map(|c| c.path.as_path()), + }, + ); + + segments.push(Segment { + audio, + cursor, + decoders, + }); + } - AudioData::from_file(audio_path).unwrap() - }); + segments + } + }; let (frame_tx, frame_rx) = tokio::sync::mpsc::channel(4); let (ws_port, ws_shutdown) = create_frames_ws(frame_rx).await; - let cursor = Arc::new(meta.cursor_data()); - let render_constants = Arc::new( - RenderVideoConstants::new( - render_options, - cursor.clone(), - project_path.clone(), // Add project path argument - ) - .await - .unwrap(), + RenderVideoConstants::new(render_options, &meta) + .await + .unwrap(), ); let renderer = Arc::new(editor::Renderer::spawn(render_constants.clone(), frame_tx)); @@ -98,13 +143,10 @@ impl EditorInstance { let this = Arc::new(Self { id: video_id, project_path, - decoders: RecordingDecoders::new(&meta), recordings, ws_port, renderer, render_constants, - audio: Arc::new(audio), - cursor, state: Arc::new(Mutex::new(EditorState { playhead_position: 0, playback_task: None, @@ -114,6 +156,7 @@ impl EditorInstance { preview_tx, project_config: watch::channel(meta.project_config()), ws_shutdown: Arc::new(StdMutex::new(Some(ws_shutdown))), + segments: Arc::new(segments), }); this.state.lock().await.preview_task = @@ -154,10 +197,6 @@ impl EditorInstance { println!("Stopping renderer"); self.renderer.stop().await; - // Stop decoders - println!("Stopping decoders"); - self.decoders.stop().await; - // // Clear audio data // if self.audio.lock().unwrap().is_some() { // println!("Clearing audio data"); @@ -187,11 +226,9 @@ impl EditorInstance { let start_frame_number = state.playhead_position; let playback_handle = playback::Playback { - audio: Arc::clone(&self.audio), + segments: self.segments.clone(), renderer: self.renderer.clone(), render_constants: self.render_constants.clone(), - decoders: self.decoders.clone(), - recordings: self.recordings, start_frame_number, project: self.project_config.0.subscribe(), } @@ -241,17 +278,21 @@ impl EditorInstance { let project = self.project_config.1.borrow().clone(); - let Some(time) = project + let Some((time, segment)) = project .timeline .as_ref() .map(|timeline| timeline.get_recording_time(frame_number as f64 / FPS as f64)) - .unwrap_or(Some(frame_number as f64 / FPS as f64)) + .unwrap_or(Some((frame_number as f64 / FPS as f64, None))) else { continue; }; - let Some((screen_frame, camera_frame)) = - self.decoders.get_frames((time * FPS as f64) as u32).await + let segment = &self.segments[segment.unwrap_or(0) as usize]; + + let Some((screen_frame, camera_frame)) = segment + .decoders + .get_frames((time * FPS as f64) as u32) + .await else { continue; }; @@ -368,3 +409,9 @@ pub enum SocketMessage { height: u32, }, } + +pub struct Segment { + pub audio: Arc>, + pub cursor: Arc, + pub decoders: RecordingSegmentDecoders, +} diff --git a/crates/editor/src/lib.rs b/crates/editor/src/lib.rs index 4329334c..f3b0532d 100644 --- a/crates/editor/src/lib.rs +++ b/crates/editor/src/lib.rs @@ -3,5 +3,5 @@ mod editor_instance; mod playback; mod project_recordings; -pub use editor_instance::{EditorInstance, EditorState, FRAMES_WS_PATH}; -pub use project_recordings::ProjectRecordings; +pub use editor_instance::{EditorInstance, EditorState, Segment, FRAMES_WS_PATH}; +pub use project_recordings::{ProjectRecordings, SegmentRecordings}; diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs index 140117cb..64a93799 100644 --- a/crates/editor/src/playback.rs +++ b/crates/editor/src/playback.rs @@ -3,23 +3,22 @@ use std::{sync::Arc, time::Duration}; use cap_media::data::{AudioInfo, FromSampleBytes}; use cap_media::feeds::{AudioData, AudioPlaybackBuffer}; use cap_project::ProjectConfiguration; -use cap_rendering::{ProjectUniforms, RecordingDecoders, RenderVideoConstants}; +use cap_rendering::{ProjectUniforms, RenderVideoConstants}; use cpal::{ traits::{DeviceTrait, HostTrait, StreamTrait}, BufferSize, SampleFormat, }; use tokio::{sync::watch, time::Instant}; -use crate::{editor, project_recordings::ProjectRecordings}; +use crate::editor; +use crate::editor_instance::Segment; pub struct Playback { - pub audio: Arc>, pub renderer: Arc, pub render_constants: Arc, - pub decoders: RecordingDecoders, pub start_frame_number: u32, pub project: watch::Receiver, - pub recordings: ProjectRecordings, + pub segments: Arc>, } const FPS: u32 = 30; @@ -63,16 +62,16 @@ impl Playback { .unwrap_or(f64::MAX); // Lock the mutex and check if audio data is available - if let Some(audio_data) = self.audio.as_ref() { - AudioPlayback { - audio: audio_data.clone(), - stop_rx: stop_rx.clone(), - start_frame_number: self.start_frame_number, - duration, - project: self.project.clone(), - } - .spawn(); - }; + // if let Some(audio_data) = self.audio.as_ref() { + // AudioPlayback { + // audio: audio_data.clone(), + // stop_rx: stop_rx.clone(), + // start_frame_number: self.start_frame_number, + // duration, + // project: self.project.clone(), + // } + // .spawn(); + // }; loop { if frame_number as f64 > FPS as f64 * duration { @@ -81,20 +80,22 @@ impl Playback { let project = self.project.borrow().clone(); - let time = if let Some(timeline) = project.timeline() { + let (time, segment) = if let Some(timeline) = project.timeline() { match timeline.get_recording_time(frame_number as f64 / FPS as f64) { Some(time) => time, None => break, } } else { - frame_number as f64 / FPS as f64 + (frame_number as f64 / FPS as f64, None) }; + let segment = &self.segments[segment.unwrap_or(0) as usize]; + tokio::select! { _ = stop_rx.changed() => { break; }, - Some((screen_frame, camera_frame)) = self.decoders.get_frames((time * FPS as f64) as u32) => { + Some((screen_frame, camera_frame)) = segment.decoders.get_frames((time * FPS as f64) as u32) => { // println!("decoded frame in {:?}", debug.elapsed()); let uniforms = ProjectUniforms::new(&self.render_constants, &project, time as f32); diff --git a/crates/editor/src/project_recordings.rs b/crates/editor/src/project_recordings.rs index 04906f6c..084c7724 100644 --- a/crates/editor/src/project_recordings.rs +++ b/crates/editor/src/project_recordings.rs @@ -56,32 +56,70 @@ impl Audio { } } -#[derive(Debug, Clone, Copy, Serialize, Type)] +#[derive(Debug, Clone, Serialize, Type)] pub struct ProjectRecordings { - pub display: Video, - pub camera: Option