From 0bbddd1e60a03fd593b1c443e993509011ff2d14 Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 8 Apr 2026 16:46:31 -0700 Subject: [PATCH] feat: allow recording with webcam overlay --- js/src/api/types.ts | 1 + js/src/electron.d.ts | 3 +- js/src/features/compose/compose-overlay.tsx | 5 + .../features/compose/use-screen-recorder.ts | 161 ++++++++++++++++-- .../particles/media-particle-view.tsx | 2 +- js/src/main.ts | 25 ++- js/src/preload.ts | 7 +- .../ScreenRecordControlApp.tsx | 67 +++++++- 8 files changed, 251 insertions(+), 20 deletions(-) diff --git a/js/src/api/types.ts b/js/src/api/types.ts index 2e99ee8..3f4fde1 100644 --- a/js/src/api/types.ts +++ b/js/src/api/types.ts @@ -121,6 +121,7 @@ export const MediaPropertiesSchema = z.object({ duration_ms: z.number(), size_bytes: z.number(), transcript: TranscriptSchema.optional(), + source: z.enum(["camera", "screen"]).optional(), }); export type MediaProperties = z.infer; diff --git a/js/src/electron.d.ts b/js/src/electron.d.ts index e8cc949..1c11cdd 100644 --- a/js/src/electron.d.ts +++ b/js/src/electron.d.ts @@ -32,12 +32,13 @@ declare global { }; electronScreen: { getScreenSources: () => Promise; - startRecordingWindow: () => void; + startRecordingWindow: (data: { includeCamera: boolean }) => void; stopRecordingWindow: () => void; onStopRequested: (callback: () => void) => () => void; }; electronScreenRecord: { stop: () => void; + onInit: (callback: (data: { includeCamera: boolean }) => void) => () => void; }; electronLink: { fetchMetadata: (url: string) => Promise; diff --git a/js/src/features/compose/compose-overlay.tsx b/js/src/features/compose/compose-overlay.tsx index 140f46c..39331b4 100644 --- a/js/src/features/compose/compose-overlay.tsx +++ b/js/src/features/compose/compose-overlay.tsx @@ -285,6 +285,7 @@ export function ComposeOverlay({ reviewMimeType, ); + const isAudioOnly = reviewMimeType.startsWith("audio/"); particleId = await createParticle.mutateAsync({ path, type: "media", @@ -293,6 +294,9 @@ export function ComposeOverlay({ mime_type: reviewMimeType, duration_ms: reviewDurationMs, size_bytes, + ...(!isAudioOnly && { + source: recordingSource === "screen" ? "screen" as const : "camera" as const, + }), }, createdByHumanId: userId, }); @@ -309,6 +313,7 @@ export function ComposeOverlay({ reviewBlob, reviewMimeType, reviewDurationMs, + recordingSource, createParticle, uploadMedia, uploadAttachments, diff --git a/js/src/features/compose/use-screen-recorder.ts b/js/src/features/compose/use-screen-recorder.ts index 40f4a1a..82bc296 100644 --- a/js/src/features/compose/use-screen-recorder.ts +++ b/js/src/features/compose/use-screen-recorder.ts @@ -1,4 +1,5 @@ import { useCallback, useEffect, useRef } from "react"; +import type { RecordingMode } from "@/stores/media-settings-store"; const VIDEO_PREFERRED_MIME = "video/webm;codecs=vp9,opus"; const VIDEO_FALLBACK_MIME = "video/webm"; @@ -9,23 +10,135 @@ function getScreenMime(): string { : VIDEO_FALLBACK_MIME; } +// --------------------------------------------------------------------------- +// Canvas compositor — overlays webcam as a circular PiP on the screen feed +// --------------------------------------------------------------------------- + +interface Compositor { + /** Composited video stream (screen + optional webcam bubble). */ + stream: MediaStream; + /** Tear down the animation loop and video elements. */ + stop: () => void; +} + +function createCompositor( + screenStream: MediaStream, + cameraStream: MediaStream | null, +): Compositor { + const screenTrack = screenStream.getVideoTracks()[0]; + const settings = screenTrack.getSettings(); + const width = settings.width ?? 1920; + const height = settings.height ?? 1080; + + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext("2d")!; + + // Hidden video elements used as frame sources + const screenVideo = document.createElement("video"); + screenVideo.srcObject = screenStream; + screenVideo.muted = true; + screenVideo.playsInline = true; + screenVideo.play(); + + let cameraVideo: HTMLVideoElement | null = null; + if (cameraStream) { + cameraVideo = document.createElement("video"); + cameraVideo.srcObject = cameraStream; + cameraVideo.muted = true; + cameraVideo.playsInline = true; + cameraVideo.play(); + } + + let animId = 0; + + const draw = () => { + // Screen — full canvas + ctx.drawImage(screenVideo, 0, 0, width, height); + + // Webcam — circular bubble in bottom-left + if (cameraVideo && cameraVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + const bubbleSize = Math.round(Math.min(width, height) * 0.18); + const margin = Math.round(bubbleSize * 0.3); + const cx = margin + bubbleSize / 2; + const cy = height - margin - bubbleSize / 2; + + ctx.save(); + ctx.beginPath(); + ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2); + ctx.clip(); + + // Crop camera to square centre, mirror horizontally + const vw = cameraVideo.videoWidth || 1; + const vh = cameraVideo.videoHeight || 1; + const side = Math.min(vw, vh); + const sx = (vw - side) / 2; + const sy = (vh - side) / 2; + + ctx.translate(cx, cy); + ctx.scale(-1, 1); // horizontal flip + ctx.drawImage( + cameraVideo, + sx, sy, side, side, + -bubbleSize / 2, -bubbleSize / 2, bubbleSize, bubbleSize, + ); + ctx.restore(); + + // Subtle ring around the bubble + ctx.beginPath(); + ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2); + ctx.strokeStyle = "rgba(255,255,255,0.25)"; + ctx.lineWidth = 2; + ctx.stroke(); + } + + animId = requestAnimationFrame(draw); + }; + draw(); + + return { + stream: canvas.captureStream(30), + stop: () => { + cancelAnimationFrame(animId); + screenVideo.pause(); + screenVideo.srcObject = null; + if (cameraVideo) { + cameraVideo.pause(); + cameraVideo.srcObject = null; + } + }, + }; +} + +// --------------------------------------------------------------------------- +// Hook +// --------------------------------------------------------------------------- + interface UseScreenRecorderOptions { + /** "video" = screen + webcam overlay + mic. "audio" = screen + mic only. */ + mode: RecordingMode; onFinish: (blob: Blob, durationMs: number, mimeType: string) => void; onError: (message: string) => void; } /** * Manages screen recording via Electron's desktopCapturer. - * Captures screen video + mic audio into a single MediaRecorder. - * Communicates with the floating control window via IPC. + * + * When mode is "video", captures screen video composited with a webcam + * overlay (via Canvas) plus mic audio. + * When mode is "audio", captures screen video with mic audio only. */ export function useScreenRecorder({ + mode, onFinish, onError, }: UseScreenRecorderOptions) { const recorderRef = useRef(null); const screenStreamRef = useRef(null); const micStreamRef = useRef(null); + const cameraStreamRef = useRef(null); + const compositorRef = useRef(null); const chunksRef = useRef([]); const startTimeRef = useRef(0); const cleanupIpcRef = useRef<(() => void) | null>(null); @@ -37,11 +150,18 @@ export function useScreenRecorder({ onErrorRef.current = onError; }); + const modeRef = useRef(mode); + modeRef.current = mode; + const stopAllTracks = useCallback(() => { + compositorRef.current?.stop(); + compositorRef.current = null; screenStreamRef.current?.getTracks().forEach((t) => t.stop()); micStreamRef.current?.getTracks().forEach((t) => t.stop()); + cameraStreamRef.current?.getTracks().forEach((t) => t.stop()); screenStreamRef.current = null; micStreamRef.current = null; + cameraStreamRef.current = null; recorderRef.current = null; chunksRef.current = []; cleanupIpcRef.current?.(); @@ -51,7 +171,9 @@ export function useScreenRecorder({ const startRecording = useCallback( async (sourceId: string) => { try { - // Screen video via Electron desktopCapturer + const includeCamera = modeRef.current === "video"; + + // 1. Screen video const screenStream = await navigator.mediaDevices.getUserMedia({ audio: false, video: { @@ -63,14 +185,31 @@ export function useScreenRecorder({ }); screenStreamRef.current = screenStream; - // Mic audio separately - const micStream = - await navigator.mediaDevices.getUserMedia({ audio: true }); + // 2. Mic audio + const micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); micStreamRef.current = micStream; - // Combine into a single stream + // 3. Camera (only in video mode) + let cameraStream: MediaStream | null = null; + if (includeCamera) { + try { + cameraStream = await navigator.mediaDevices.getUserMedia({ + video: { aspectRatio: { ideal: 1 }, width: { ideal: 320 } }, + audio: false, + }); + cameraStreamRef.current = cameraStream; + } catch { + // Camera unavailable — proceed without it + } + } + + // 4. Composite screen + camera via canvas + const compositor = createCompositor(screenStream, cameraStream); + compositorRef.current = compositor; + + // 5. Combine composited video + mic audio const combined = new MediaStream([ - ...screenStream.getVideoTracks(), + ...compositor.stream.getVideoTracks(), ...micStream.getAudioTracks(), ]); @@ -98,10 +237,10 @@ export function useScreenRecorder({ recorder.start(); - // Show the floating control window - window.electronScreen.startRecordingWindow(); + // 6. Show floating control window + window.electronScreen.startRecordingWindow({ includeCamera }); - // Listen for stop from floating window + // 7. Listen for stop from floating window cleanupIpcRef.current = window.electronScreen.onStopRequested(() => { if (recorderRef.current?.state === "recording") { recorderRef.current.stop(); diff --git a/js/src/features/particles/media-particle-view.tsx b/js/src/features/particles/media-particle-view.tsx index 990b6b1..78cdcaa 100644 --- a/js/src/features/particles/media-particle-view.tsx +++ b/js/src/features/particles/media-particle-view.tsx @@ -147,7 +147,7 @@ export const MediaParticleView = forwardRef {transcript && ( diff --git a/js/src/main.ts b/js/src/main.ts index 249824d..86fbd57 100644 --- a/js/src/main.ts +++ b/js/src/main.ts @@ -209,9 +209,30 @@ ipcMain.handle('screen:get-sources', async () => { }); // Screen recording IPC handlers -ipcMain.on('screen-record:start', () => { +ipcMain.on('screen-record:start', (_event, data: { includeCamera: boolean }) => { createScreenRecordWindow(); - screenRecordWindow?.showInactive(); + if (!screenRecordWindow) return; + + // Resize based on whether camera preview is shown + const winW = data.includeCamera ? 200 : 240; + const winH = data.includeCamera ? 176 : 48; + const { width, height } = screen.getPrimaryDisplay().workAreaSize; + screenRecordWindow.setSize(winW, winH); + screenRecordWindow.setPosition( + Math.round((width - winW) / 2), + height - winH - 32, + ); + + const send = () => { + screenRecordWindow?.webContents.send('screen-record:init', data); + screenRecordWindow?.showInactive(); + }; + + if (screenRecordWindow.webContents.isLoading()) { + screenRecordWindow.webContents.once('did-finish-load', send); + } else { + send(); + } }); ipcMain.on('screen-record:stop', () => { mainWindow?.webContents.send('screen-record:stopped'); diff --git a/js/src/preload.ts b/js/src/preload.ts index ef5f076..5f3579e 100644 --- a/js/src/preload.ts +++ b/js/src/preload.ts @@ -43,7 +43,7 @@ contextBridge.exposeInMainWorld('electronAutoplay', { contextBridge.exposeInMainWorld('electronScreen', { getScreenSources: () => ipcRenderer.invoke('screen:get-sources'), - startRecordingWindow: () => ipcRenderer.send('screen-record:start'), + startRecordingWindow: (data: { includeCamera: boolean }) => ipcRenderer.send('screen-record:start', data), stopRecordingWindow: () => ipcRenderer.send('screen-record:cancel'), onStopRequested: (callback: () => void) => { const handler = () => callback(); @@ -54,6 +54,11 @@ contextBridge.exposeInMainWorld('electronScreen', { contextBridge.exposeInMainWorld('electronScreenRecord', { stop: () => ipcRenderer.send('screen-record:stop'), + onInit: (callback: (data: { includeCamera: boolean }) => void) => { + const handler = (_event: Electron.IpcRendererEvent, data: { includeCamera: boolean }) => callback(data); + ipcRenderer.on('screen-record:init', handler); + return () => { ipcRenderer.removeListener('screen-record:init', handler); }; + }, }); contextBridge.exposeInMainWorld('electronLink', { diff --git a/js/src/screen_record_window/ScreenRecordControlApp.tsx b/js/src/screen_record_window/ScreenRecordControlApp.tsx index 41cac98..c3f6c16 100644 --- a/js/src/screen_record_window/ScreenRecordControlApp.tsx +++ b/js/src/screen_record_window/ScreenRecordControlApp.tsx @@ -1,9 +1,48 @@ -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { Square } from "lucide-react"; export function ScreenRecordControlApp() { const [elapsed, setElapsed] = useState(0); + const [includeCamera, setIncludeCamera] = useState(false); + const videoRef = useRef(null); + const streamRef = useRef(null); + // Listen for init data from main process + useEffect(() => { + return window.electronScreenRecord.onInit((data) => { + setIncludeCamera(data.includeCamera); + }); + }, []); + + // Acquire webcam for preview (independent from the recording capture) + useEffect(() => { + if (!includeCamera) return; + + let cancelled = false; + navigator.mediaDevices + .getUserMedia({ video: { aspectRatio: { ideal: 1 }, width: { ideal: 160 } }, audio: false }) + .then((stream) => { + if (cancelled) { + stream.getTracks().forEach((t) => t.stop()); + return; + } + streamRef.current = stream; + if (videoRef.current) { + videoRef.current.srcObject = stream; + } + }) + .catch(() => { + // Camera unavailable — just don't show preview + }); + + return () => { + cancelled = true; + streamRef.current?.getTracks().forEach((t) => t.stop()); + streamRef.current = null; + }; + }, [includeCamera]); + + // Timer useEffect(() => { const interval = setInterval(() => setElapsed((prev) => prev + 1), 1000); return () => clearInterval(interval); @@ -13,13 +52,33 @@ export function ScreenRecordControlApp() { const seconds = elapsed % 60; const display = `${minutes}:${seconds.toString().padStart(2, "0")}`; + const handleStop = useCallback(() => { + streamRef.current?.getTracks().forEach((t) => t.stop()); + streamRef.current = null; + window.electronScreenRecord.stop(); + }, []); + return ( -
-
+
+ {/* Webcam preview */} + {includeCamera && ( +
+
+ )} + + {/* Controls */} +
{display}