FlowyLabs/llink
1
0
Fork 0
Code Issues 71 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: freezes during longer screen recordings

Browse Source 
Closes #131

Fundamentally, we cannot afford to composite webcam and screen on the
client. We should create two streams and then combine them with ffmpeg
processing in the backend / orion.
This commit is contained in:
talksik
2026-04-09 08:15:27 -07:00
parent 5fadc6c8fe
commit f91068a4c1
6 changed files with 28 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/src/electron.d.ts
Vendored
+2 -2
View File
@@ -32,13 +32,13 @@ declare global {
};
electronScreen: {
getScreenSources: () => Promise<ScreenSource[]>;
startRecordingWindow: (data: { includeCamera: boolean }) => void;
startRecordingWindow: () => void;
stopRecordingWindow: () => void;
onStopRequested: (callback: () => void) => () => void;
};
electronScreenRecord: {
stop: () => void;
onInit: (callback: (data: { includeCamera: boolean }) => void) => () => void;
onInit: (callback: () => void) => () => void;
};
electronLink: {
fetchMetadata: (url: string) => Promise<LinkMetadata | null>;
js/src/features/compose/compose-overlay.tsx
-1
View File
@@ -164,7 +164,6 @@ export function ComposeOverlay({
stopRecording: stopScreenRecording,
cancelRecording: cancelScreenRecording,
} = useScreenRecorder({
mode: recordingMode,
onFinish: (blob, durationMs, mimeType) => {
setStepSync("reviewing");
setReviewBlob(blob);
js/src/features/compose/use-screen-recorder.ts
+8 -145
View File
@@ -1,5 +1,4 @@
import { useCallback, useEffect, useRef } from "react";
import type { RecordingMode } from "@/stores/media-settings-store";
const VIDEO_PREFERRED_MIME = "video/webm;codecs=vp9,opus";
const VIDEO_FALLBACK_MIME = "video/webm";
@@ -10,135 +9,26 @@ function getScreenMime(): string {
: VIDEO_FALLBACK_MIME;
}
// ---------------------------------------------------------------------------
// Canvas compositor — overlays webcam as a circular PiP on the screen feed
// ---------------------------------------------------------------------------
interface Compositor {
/** Composited video stream (screen + optional webcam bubble). */
stream: MediaStream;
/** Tear down the animation loop and video elements. */
stop: () => void;
}
function createCompositor(
screenStream: MediaStream,
cameraStream: MediaStream | null,
): Compositor {
const screenTrack = screenStream.getVideoTracks()[0];
const settings = screenTrack.getSettings();
const width = settings.width ?? 1920;
const height = settings.height ?? 1080;
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext("2d")!;
// Hidden video elements used as frame sources
const screenVideo = document.createElement("video");
screenVideo.srcObject = screenStream;
screenVideo.muted = true;
screenVideo.playsInline = true;
screenVideo.play();
let cameraVideo: HTMLVideoElement | null = null;
if (cameraStream) {
cameraVideo = document.createElement("video");
cameraVideo.srcObject = cameraStream;
cameraVideo.muted = true;
cameraVideo.playsInline = true;
cameraVideo.play();
}
let animId = 0;
const draw = () => {
// Screen — full canvas
ctx.drawImage(screenVideo, 0, 0, width, height);
// Webcam — circular bubble in bottom-left
if (cameraVideo && cameraVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
const bubbleSize = Math.round(Math.min(width, height) * 0.18);
const margin = Math.round(bubbleSize * 0.3);
const cx = margin + bubbleSize / 2;
const cy = height - margin - bubbleSize / 2;
ctx.save();
ctx.beginPath();
ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2);
ctx.clip();
// Crop camera to square centre, mirror horizontally
const vw = cameraVideo.videoWidth || 1;
const vh = cameraVideo.videoHeight || 1;
const side = Math.min(vw, vh);
const sx = (vw - side) / 2;
const sy = (vh - side) / 2;
ctx.translate(cx, cy);
ctx.scale(-1, 1); // horizontal flip
ctx.drawImage(
cameraVideo,
sx, sy, side, side,
-bubbleSize / 2, -bubbleSize / 2, bubbleSize, bubbleSize,
);
ctx.restore();
// Subtle ring around the bubble
ctx.beginPath();
ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2);
ctx.strokeStyle = "rgba(255,255,255,0.25)";
ctx.lineWidth = 2;
ctx.stroke();
}
animId = requestAnimationFrame(draw);
};
draw();
return {
stream: canvas.captureStream(30),
stop: () => {
cancelAnimationFrame(animId);
screenVideo.pause();
screenVideo.srcObject = null;
if (cameraVideo) {
cameraVideo.pause();
cameraVideo.srcObject = null;
}
},
};
}
// ---------------------------------------------------------------------------
// Hook
// ---------------------------------------------------------------------------
interface UseScreenRecorderOptions {
/** "video" = screen + webcam overlay + mic. "audio" = screen + mic only. */
mode: RecordingMode;
onFinish: (blob: Blob, durationMs: number, mimeType: string) => void;
onError: (message: string) => void;
}
/**
* Manages screen recording via Electron's desktopCapturer.
*
* When mode is "video", captures screen video composited with a webcam
* overlay (via Canvas) plus mic audio.
* When mode is "audio", captures screen video with mic audio only.
* Captures screen video + mic audio.
*/
export function useScreenRecorder({
mode,
onFinish,
onError,
}: UseScreenRecorderOptions) {
const recorderRef = useRef<MediaRecorder | null>(null);
const screenStreamRef = useRef<MediaStream | null>(null);
const micStreamRef = useRef<MediaStream | null>(null);
const cameraStreamRef = useRef<MediaStream | null>(null);
const compositorRef = useRef<Compositor | null>(null);
const chunksRef = useRef<Blob[]>([]);
const startTimeRef = useRef<number>(0);
const cleanupIpcRef = useRef<(() => void) | null>(null);
@@ -150,18 +40,11 @@ export function useScreenRecorder({
onErrorRef.current = onError;
});
const modeRef = useRef(mode);
modeRef.current = mode;
const stopAllTracks = useCallback(() => {
compositorRef.current?.stop();
compositorRef.current = null;
screenStreamRef.current?.getTracks().forEach((t) => t.stop());
micStreamRef.current?.getTracks().forEach((t) => t.stop());
cameraStreamRef.current?.getTracks().forEach((t) => t.stop());
screenStreamRef.current = null;
micStreamRef.current = null;
cameraStreamRef.current = null;
recorderRef.current = null;
chunksRef.current = [];
cleanupIpcRef.current?.();
@@ -171,8 +54,6 @@ export function useScreenRecorder({
const startRecording = useCallback(
async (sourceId: string) => {
try {
const includeCamera = modeRef.current === "video";
// 1. Screen video
const screenStream = await navigator.mediaDevices.getUserMedia({
audio: false,
@@ -181,7 +62,7 @@ export function useScreenRecorder({
chromeMediaSource: "desktop",
chromeMediaSourceId: sourceId,
},
} as unknown as MediaTrackConstraints,
} as MediaTrackConstraints,
});
screenStreamRef.current = screenStream;
@@ -189,27 +70,9 @@ export function useScreenRecorder({
const micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
micStreamRef.current = micStream;
// 3. Camera (only in video mode)
let cameraStream: MediaStream | null = null;
if (includeCamera) {
try {
cameraStream = await navigator.mediaDevices.getUserMedia({
video: { aspectRatio: { ideal: 1 }, width: { ideal: 320 } },
audio: false,
});
cameraStreamRef.current = cameraStream;
} catch {
// Camera unavailable — proceed without it
}
}
// 4. Composite screen + camera via canvas
const compositor = createCompositor(screenStream, cameraStream);
compositorRef.current = compositor;
// 5. Combine composited video + mic audio
// 3. Combine screen video + mic audio
const combined = new MediaStream([
...compositor.stream.getVideoTracks(),
...screenStream.getVideoTracks(),
...micStream.getAudioTracks(),
]);
@@ -235,12 +98,12 @@ export function useScreenRecorder({
}
};
recorder.start();
recorder.start(1000);
// 6. Show floating control window
window.electronScreen.startRecordingWindow({ includeCamera });
// 4. Show floating control window
window.electronScreen.startRecordingWindow();
// 7. Listen for stop from floating window
// 5. Listen for stop from floating window
cleanupIpcRef.current = window.electronScreen.onStopRequested(() => {
if (recorderRef.current?.state === "recording") {
recorderRef.current.stop();
js/src/main.ts
+4 -5
View File
@@ -209,13 +209,12 @@ ipcMain.handle('screen:get-sources', async () => {
});
// Screen recording IPC handlers
ipcMain.on('screen-record:start', (_event, data: { includeCamera: boolean }) => {
ipcMain.on('screen-record:start', () => {
createScreenRecordWindow();
if (!screenRecordWindow) return;
// Resize based on whether camera preview is shown
const winW = data.includeCamera ? 200 : 240;
const winH = data.includeCamera ? 176 : 48;
const winW = 240;
const winH = 48;
const { width, height } = screen.getPrimaryDisplay().workAreaSize;
screenRecordWindow.setSize(winW, winH);
screenRecordWindow.setPosition(
@@ -224,7 +223,7 @@ ipcMain.on('screen-record:start', (_event, data: { includeCamera: boolean }) =>
);
const send = () => {
screenRecordWindow?.webContents.send('screen-record:init', data);
screenRecordWindow?.webContents.send('screen-record:init');
screenRecordWindow?.showInactive();
};
js/src/preload.ts
+3 -3
View File
@@ -43,7 +43,7 @@ contextBridge.exposeInMainWorld('electronAutoplay', {
contextBridge.exposeInMainWorld('electronScreen', {
getScreenSources: () => ipcRenderer.invoke('screen:get-sources'),
startRecordingWindow: (data: { includeCamera: boolean }) => ipcRenderer.send('screen-record:start', data),
startRecordingWindow: () => ipcRenderer.send('screen-record:start'),
stopRecordingWindow: () => ipcRenderer.send('screen-record:cancel'),
onStopRequested: (callback: () => void) => {
const handler = () => callback();
@@ -54,8 +54,8 @@ contextBridge.exposeInMainWorld('electronScreen', {
contextBridge.exposeInMainWorld('electronScreenRecord', {
stop: () => ipcRenderer.send('screen-record:stop'),
onInit: (callback: (data: { includeCamera: boolean }) => void) => {
const handler = (_event: Electron.IpcRendererEvent, data: { includeCamera: boolean }) => callback(data);
onInit: (callback: () => void) => {
const handler = () => callback();
ipcRenderer.on('screen-record:init', handler);
return () => { ipcRenderer.removeListener('screen-record:init', handler); };
},
js/src/screen_record_window/ScreenRecordControlApp.tsx
+11 -67
View File
@@ -1,46 +1,8 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { useCallback, useEffect, useState } from "react";
import { Square } from "lucide-react";
export function ScreenRecordControlApp() {
const [elapsed, setElapsed] = useState(0);
const [includeCamera, setIncludeCamera] = useState(false);
const videoRef = useRef<HTMLVideoElement>(null);
const streamRef = useRef<MediaStream | null>(null);
// Listen for init data from main process
useEffect(() => {
return window.electronScreenRecord.onInit((data) => {
setIncludeCamera(data.includeCamera);
});
}, []);
// Acquire webcam for preview (independent from the recording capture)
useEffect(() => {
if (!includeCamera) return;
let cancelled = false;
navigator.mediaDevices
.getUserMedia({ video: { aspectRatio: { ideal: 1 }, width: { ideal: 160 } }, audio: false })
.then((stream) => {
if (cancelled) {
stream.getTracks().forEach((t) => t.stop());
return;
}
streamRef.current = stream;
if (videoRef.current) {
videoRef.current.srcObject = stream;
}
})
.catch(() => {
// Camera unavailable — just don't show preview
});
return () => {
cancelled = true;
streamRef.current?.getTracks().forEach((t) => t.stop());
streamRef.current = null;
};
}, [includeCamera]);
// Timer
useEffect(() => {
@@ -53,38 +15,20 @@ export function ScreenRecordControlApp() {
const display = `${minutes}:${seconds.toString().padStart(2, "0")}`;
const handleStop = useCallback(() => {
streamRef.current?.getTracks().forEach((t) => t.stop());
streamRef.current = null;
window.electronScreenRecord.stop();
}, []);
return (
<div className="flex h-screen w-screen flex-col items-center justify-center gap-2 bg-zinc-900 px-4 py-3">
{/* Webcam preview */}
{includeCamera && (
<div className="relative h-24 w-24 overflow-hidden rounded-full bg-zinc-800">
<video
ref={videoRef}
autoPlay
playsInline
muted
className="h-full w-full -scale-x-100 object-cover"
/>
</div>
)}
{/* Controls */}
<div className="flex items-center gap-3">
<span className="h-2.5 w-2.5 animate-pulse rounded-full bg-red-500" />
<span className="font-mono text-sm text-white/80">{display}</span>
<button
onClick={handleStop}
className="flex items-center gap-1.5 rounded-md bg-red-600 px-3 py-1 text-xs font-medium text-white transition-colors hover:bg-red-500"
>
<Square className="size-3 fill-current" />
Stop
</button>
</div>
<div className="flex h-screen w-screen items-center justify-center gap-3 bg-zinc-900 px-4 py-3">
<span className="h-2.5 w-2.5 animate-pulse rounded-full bg-red-500" />
<span className="font-mono text-sm text-white/80">{display}</span>
<button
onClick={handleStop}
className="flex items-center gap-1.5 rounded-md bg-red-600 px-3 py-1 text-xs font-medium text-white transition-colors hover:bg-red-500"
>
<Square className="size-3 fill-current" />
Stop
</button>
</div>
);
}