From f91068a4c15068047c973ecb8045fb28613aafe8 Mon Sep 17 00:00:00 2001 From: talksik Date: Thu, 9 Apr 2026 08:15:27 -0700 Subject: [PATCH] fix: freezes during longer screen recordings Closes #131 Fundamentally, we cannot afford to composite webcam and screen on the client. We should create two streams and then combine them with ffmpeg processing in the backend / orion. --- js/src/electron.d.ts | 4 +- js/src/features/compose/compose-overlay.tsx | 1 - .../features/compose/use-screen-recorder.ts | 153 +----------------- js/src/main.ts | 9 +- js/src/preload.ts | 6 +- .../ScreenRecordControlApp.tsx | 78 ++------- 6 files changed, 28 insertions(+), 223 deletions(-) diff --git a/js/src/electron.d.ts b/js/src/electron.d.ts index 1c11cdd..16a096a 100644 --- a/js/src/electron.d.ts +++ b/js/src/electron.d.ts @@ -32,13 +32,13 @@ declare global { }; electronScreen: { getScreenSources: () => Promise; - startRecordingWindow: (data: { includeCamera: boolean }) => void; + startRecordingWindow: () => void; stopRecordingWindow: () => void; onStopRequested: (callback: () => void) => () => void; }; electronScreenRecord: { stop: () => void; - onInit: (callback: (data: { includeCamera: boolean }) => void) => () => void; + onInit: (callback: () => void) => () => void; }; electronLink: { fetchMetadata: (url: string) => Promise; diff --git a/js/src/features/compose/compose-overlay.tsx b/js/src/features/compose/compose-overlay.tsx index 144e874..80139eb 100644 --- a/js/src/features/compose/compose-overlay.tsx +++ b/js/src/features/compose/compose-overlay.tsx @@ -164,7 +164,6 @@ export function ComposeOverlay({ stopRecording: stopScreenRecording, cancelRecording: cancelScreenRecording, } = useScreenRecorder({ - mode: recordingMode, onFinish: (blob, durationMs, mimeType) => { setStepSync("reviewing"); setReviewBlob(blob); diff --git a/js/src/features/compose/use-screen-recorder.ts b/js/src/features/compose/use-screen-recorder.ts index 82bc296..07689fd 100644 --- a/js/src/features/compose/use-screen-recorder.ts +++ b/js/src/features/compose/use-screen-recorder.ts @@ -1,5 +1,4 @@ import { useCallback, useEffect, useRef } from "react"; -import type { RecordingMode } from "@/stores/media-settings-store"; const VIDEO_PREFERRED_MIME = "video/webm;codecs=vp9,opus"; const VIDEO_FALLBACK_MIME = "video/webm"; @@ -10,135 +9,26 @@ function getScreenMime(): string { : VIDEO_FALLBACK_MIME; } -// --------------------------------------------------------------------------- -// Canvas compositor — overlays webcam as a circular PiP on the screen feed -// --------------------------------------------------------------------------- - -interface Compositor { - /** Composited video stream (screen + optional webcam bubble). */ - stream: MediaStream; - /** Tear down the animation loop and video elements. */ - stop: () => void; -} - -function createCompositor( - screenStream: MediaStream, - cameraStream: MediaStream | null, -): Compositor { - const screenTrack = screenStream.getVideoTracks()[0]; - const settings = screenTrack.getSettings(); - const width = settings.width ?? 1920; - const height = settings.height ?? 1080; - - const canvas = document.createElement("canvas"); - canvas.width = width; - canvas.height = height; - const ctx = canvas.getContext("2d")!; - - // Hidden video elements used as frame sources - const screenVideo = document.createElement("video"); - screenVideo.srcObject = screenStream; - screenVideo.muted = true; - screenVideo.playsInline = true; - screenVideo.play(); - - let cameraVideo: HTMLVideoElement | null = null; - if (cameraStream) { - cameraVideo = document.createElement("video"); - cameraVideo.srcObject = cameraStream; - cameraVideo.muted = true; - cameraVideo.playsInline = true; - cameraVideo.play(); - } - - let animId = 0; - - const draw = () => { - // Screen — full canvas - ctx.drawImage(screenVideo, 0, 0, width, height); - - // Webcam — circular bubble in bottom-left - if (cameraVideo && cameraVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { - const bubbleSize = Math.round(Math.min(width, height) * 0.18); - const margin = Math.round(bubbleSize * 0.3); - const cx = margin + bubbleSize / 2; - const cy = height - margin - bubbleSize / 2; - - ctx.save(); - ctx.beginPath(); - ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2); - ctx.clip(); - - // Crop camera to square centre, mirror horizontally - const vw = cameraVideo.videoWidth || 1; - const vh = cameraVideo.videoHeight || 1; - const side = Math.min(vw, vh); - const sx = (vw - side) / 2; - const sy = (vh - side) / 2; - - ctx.translate(cx, cy); - ctx.scale(-1, 1); // horizontal flip - ctx.drawImage( - cameraVideo, - sx, sy, side, side, - -bubbleSize / 2, -bubbleSize / 2, bubbleSize, bubbleSize, - ); - ctx.restore(); - - // Subtle ring around the bubble - ctx.beginPath(); - ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2); - ctx.strokeStyle = "rgba(255,255,255,0.25)"; - ctx.lineWidth = 2; - ctx.stroke(); - } - - animId = requestAnimationFrame(draw); - }; - draw(); - - return { - stream: canvas.captureStream(30), - stop: () => { - cancelAnimationFrame(animId); - screenVideo.pause(); - screenVideo.srcObject = null; - if (cameraVideo) { - cameraVideo.pause(); - cameraVideo.srcObject = null; - } - }, - }; -} - // --------------------------------------------------------------------------- // Hook // --------------------------------------------------------------------------- interface UseScreenRecorderOptions { - /** "video" = screen + webcam overlay + mic. "audio" = screen + mic only. */ - mode: RecordingMode; onFinish: (blob: Blob, durationMs: number, mimeType: string) => void; onError: (message: string) => void; } /** * Manages screen recording via Electron's desktopCapturer. - * - * When mode is "video", captures screen video composited with a webcam - * overlay (via Canvas) plus mic audio. - * When mode is "audio", captures screen video with mic audio only. + * Captures screen video + mic audio. */ export function useScreenRecorder({ - mode, onFinish, onError, }: UseScreenRecorderOptions) { const recorderRef = useRef(null); const screenStreamRef = useRef(null); const micStreamRef = useRef(null); - const cameraStreamRef = useRef(null); - const compositorRef = useRef(null); const chunksRef = useRef([]); const startTimeRef = useRef(0); const cleanupIpcRef = useRef<(() => void) | null>(null); @@ -150,18 +40,11 @@ export function useScreenRecorder({ onErrorRef.current = onError; }); - const modeRef = useRef(mode); - modeRef.current = mode; - const stopAllTracks = useCallback(() => { - compositorRef.current?.stop(); - compositorRef.current = null; screenStreamRef.current?.getTracks().forEach((t) => t.stop()); micStreamRef.current?.getTracks().forEach((t) => t.stop()); - cameraStreamRef.current?.getTracks().forEach((t) => t.stop()); screenStreamRef.current = null; micStreamRef.current = null; - cameraStreamRef.current = null; recorderRef.current = null; chunksRef.current = []; cleanupIpcRef.current?.(); @@ -171,8 +54,6 @@ export function useScreenRecorder({ const startRecording = useCallback( async (sourceId: string) => { try { - const includeCamera = modeRef.current === "video"; - // 1. Screen video const screenStream = await navigator.mediaDevices.getUserMedia({ audio: false, @@ -181,7 +62,7 @@ export function useScreenRecorder({ chromeMediaSource: "desktop", chromeMediaSourceId: sourceId, }, - } as unknown as MediaTrackConstraints, + } as MediaTrackConstraints, }); screenStreamRef.current = screenStream; @@ -189,27 +70,9 @@ export function useScreenRecorder({ const micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); micStreamRef.current = micStream; - // 3. Camera (only in video mode) - let cameraStream: MediaStream | null = null; - if (includeCamera) { - try { - cameraStream = await navigator.mediaDevices.getUserMedia({ - video: { aspectRatio: { ideal: 1 }, width: { ideal: 320 } }, - audio: false, - }); - cameraStreamRef.current = cameraStream; - } catch { - // Camera unavailable — proceed without it - } - } - - // 4. Composite screen + camera via canvas - const compositor = createCompositor(screenStream, cameraStream); - compositorRef.current = compositor; - - // 5. Combine composited video + mic audio + // 3. Combine screen video + mic audio const combined = new MediaStream([ - ...compositor.stream.getVideoTracks(), + ...screenStream.getVideoTracks(), ...micStream.getAudioTracks(), ]); @@ -235,12 +98,12 @@ export function useScreenRecorder({ } }; - recorder.start(); + recorder.start(1000); - // 6. Show floating control window - window.electronScreen.startRecordingWindow({ includeCamera }); + // 4. Show floating control window + window.electronScreen.startRecordingWindow(); - // 7. Listen for stop from floating window + // 5. Listen for stop from floating window cleanupIpcRef.current = window.electronScreen.onStopRequested(() => { if (recorderRef.current?.state === "recording") { recorderRef.current.stop(); diff --git a/js/src/main.ts b/js/src/main.ts index 86fbd57..007bdf4 100644 --- a/js/src/main.ts +++ b/js/src/main.ts @@ -209,13 +209,12 @@ ipcMain.handle('screen:get-sources', async () => { }); // Screen recording IPC handlers -ipcMain.on('screen-record:start', (_event, data: { includeCamera: boolean }) => { +ipcMain.on('screen-record:start', () => { createScreenRecordWindow(); if (!screenRecordWindow) return; - // Resize based on whether camera preview is shown - const winW = data.includeCamera ? 200 : 240; - const winH = data.includeCamera ? 176 : 48; + const winW = 240; + const winH = 48; const { width, height } = screen.getPrimaryDisplay().workAreaSize; screenRecordWindow.setSize(winW, winH); screenRecordWindow.setPosition( @@ -224,7 +223,7 @@ ipcMain.on('screen-record:start', (_event, data: { includeCamera: boolean }) => ); const send = () => { - screenRecordWindow?.webContents.send('screen-record:init', data); + screenRecordWindow?.webContents.send('screen-record:init'); screenRecordWindow?.showInactive(); }; diff --git a/js/src/preload.ts b/js/src/preload.ts index 5f3579e..2e592d5 100644 --- a/js/src/preload.ts +++ b/js/src/preload.ts @@ -43,7 +43,7 @@ contextBridge.exposeInMainWorld('electronAutoplay', { contextBridge.exposeInMainWorld('electronScreen', { getScreenSources: () => ipcRenderer.invoke('screen:get-sources'), - startRecordingWindow: (data: { includeCamera: boolean }) => ipcRenderer.send('screen-record:start', data), + startRecordingWindow: () => ipcRenderer.send('screen-record:start'), stopRecordingWindow: () => ipcRenderer.send('screen-record:cancel'), onStopRequested: (callback: () => void) => { const handler = () => callback(); @@ -54,8 +54,8 @@ contextBridge.exposeInMainWorld('electronScreen', { contextBridge.exposeInMainWorld('electronScreenRecord', { stop: () => ipcRenderer.send('screen-record:stop'), - onInit: (callback: (data: { includeCamera: boolean }) => void) => { - const handler = (_event: Electron.IpcRendererEvent, data: { includeCamera: boolean }) => callback(data); + onInit: (callback: () => void) => { + const handler = () => callback(); ipcRenderer.on('screen-record:init', handler); return () => { ipcRenderer.removeListener('screen-record:init', handler); }; }, diff --git a/js/src/screen_record_window/ScreenRecordControlApp.tsx b/js/src/screen_record_window/ScreenRecordControlApp.tsx index c3f6c16..b7c56eb 100644 --- a/js/src/screen_record_window/ScreenRecordControlApp.tsx +++ b/js/src/screen_record_window/ScreenRecordControlApp.tsx @@ -1,46 +1,8 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useState } from "react"; import { Square } from "lucide-react"; export function ScreenRecordControlApp() { const [elapsed, setElapsed] = useState(0); - const [includeCamera, setIncludeCamera] = useState(false); - const videoRef = useRef(null); - const streamRef = useRef(null); - - // Listen for init data from main process - useEffect(() => { - return window.electronScreenRecord.onInit((data) => { - setIncludeCamera(data.includeCamera); - }); - }, []); - - // Acquire webcam for preview (independent from the recording capture) - useEffect(() => { - if (!includeCamera) return; - - let cancelled = false; - navigator.mediaDevices - .getUserMedia({ video: { aspectRatio: { ideal: 1 }, width: { ideal: 160 } }, audio: false }) - .then((stream) => { - if (cancelled) { - stream.getTracks().forEach((t) => t.stop()); - return; - } - streamRef.current = stream; - if (videoRef.current) { - videoRef.current.srcObject = stream; - } - }) - .catch(() => { - // Camera unavailable — just don't show preview - }); - - return () => { - cancelled = true; - streamRef.current?.getTracks().forEach((t) => t.stop()); - streamRef.current = null; - }; - }, [includeCamera]); // Timer useEffect(() => { @@ -53,38 +15,20 @@ export function ScreenRecordControlApp() { const display = `${minutes}:${seconds.toString().padStart(2, "0")}`; const handleStop = useCallback(() => { - streamRef.current?.getTracks().forEach((t) => t.stop()); - streamRef.current = null; window.electronScreenRecord.stop(); }, []); return ( -
- {/* Webcam preview */} - {includeCamera && ( -
-
- )} - - {/* Controls */} -
- - {display} - -
+
+ + {display} +
); }