FlowyLabs/llink
1
0
Fork 0
Code Issues 71 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: freezes during longer screen recordings

Browse Source 
Closes #131

Fundamentally, we cannot afford to composite webcam and screen on the
client. We should create two streams and then combine them with ffmpeg
processing in the backend / orion.
This commit is contained in:
talksik
2026-04-09 08:31:23 -07:00
parent 5fadc6c8fe
commit f91068a4c1
6 changed files with 28 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/src/features/compose/compose-overlay.tsx
-1
View File
@@ -164,7 +164,6 @@ export function ComposeOverlay({
stopRecording: stopScreenRecording,
cancelRecording: cancelScreenRecording,
} = useScreenRecorder({
mode: recordingMode,
onFinish: (blob, durationMs, mimeType) => {
setStepSync("reviewing");
setReviewBlob(blob);
js/src/features/compose/use-screen-recorder.ts
+8 -145
View File
@@ -1,5 +1,4 @@
import { useCallback, useEffect, useRef } from "react";
import type { RecordingMode } from "@/stores/media-settings-store";
const VIDEO_PREFERRED_MIME = "video/webm;codecs=vp9,opus";
const VIDEO_FALLBACK_MIME = "video/webm";
@@ -10,135 +9,26 @@ function getScreenMime(): string {
: VIDEO_FALLBACK_MIME;
}
// ---------------------------------------------------------------------------
// Canvas compositor — overlays webcam as a circular PiP on the screen feed
// ---------------------------------------------------------------------------
interface Compositor {
/** Composited video stream (screen + optional webcam bubble). */
stream: MediaStream;
/** Tear down the animation loop and video elements. */
stop: () => void;
}
function createCompositor(
screenStream: MediaStream,
cameraStream: MediaStream | null,
): Compositor {
const screenTrack = screenStream.getVideoTracks()[0];
const settings = screenTrack.getSettings();
const width = settings.width ?? 1920;
const height = settings.height ?? 1080;
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext("2d")!;
// Hidden video elements used as frame sources
const screenVideo = document.createElement("video");
screenVideo.srcObject = screenStream;
screenVideo.muted = true;
screenVideo.playsInline = true;
screenVideo.play();
let cameraVideo: HTMLVideoElement | null = null;
if (cameraStream) {
cameraVideo = document.createElement("video");
cameraVideo.srcObject = cameraStream;
cameraVideo.muted = true;
cameraVideo.playsInline = true;
cameraVideo.play();
}
let animId = 0;
const draw = () => {
// Screen — full canvas
ctx.drawImage(screenVideo, 0, 0, width, height);
// Webcam — circular bubble in bottom-left
if (cameraVideo && cameraVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
const bubbleSize = Math.round(Math.min(width, height) * 0.18);
const margin = Math.round(bubbleSize * 0.3);
const cx = margin + bubbleSize / 2;
const cy = height - margin - bubbleSize / 2;
ctx.save();
ctx.beginPath();
ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2);
ctx.clip();
// Crop camera to square centre, mirror horizontally
const vw = cameraVideo.videoWidth || 1;
const vh = cameraVideo.videoHeight || 1;
const side = Math.min(vw, vh);
const sx = (vw - side) / 2;
const sy = (vh - side) / 2;
ctx.translate(cx, cy);
ctx.scale(-1, 1); // horizontal flip
ctx.drawImage(
cameraVideo,
sx, sy, side, side,
-bubbleSize / 2, -bubbleSize / 2, bubbleSize, bubbleSize,
);
ctx.restore();
// Subtle ring around the bubble
ctx.beginPath();
ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2);
ctx.strokeStyle = "rgba(255,255,255,0.25)";
ctx.lineWidth = 2;
ctx.stroke();
}
animId = requestAnimationFrame(draw);
};
draw();
return {
stream: canvas.captureStream(30),
stop: () => {
cancelAnimationFrame(animId);
screenVideo.pause();
screenVideo.srcObject = null;
if (cameraVideo) {
cameraVideo.pause();
cameraVideo.srcObject = null;
}
},
};
}
// ---------------------------------------------------------------------------
// Hook
// ---------------------------------------------------------------------------
interface UseScreenRecorderOptions {
/** "video" = screen + webcam overlay + mic. "audio" = screen + mic only. */
mode: RecordingMode;
onFinish: (blob: Blob, durationMs: number, mimeType: string) => void;
onError: (message: string) => void;
}
/**
* Manages screen recording via Electron's desktopCapturer.
*
* When mode is "video", captures screen video composited with a webcam
* overlay (via Canvas) plus mic audio.
* When mode is "audio", captures screen video with mic audio only.
* Captures screen video + mic audio.
*/
export function useScreenRecorder({
mode,
onFinish,
onError,
}: UseScreenRecorderOptions) {
const recorderRef = useRef<MediaRecorder | null>(null);
const screenStreamRef = useRef<MediaStream | null>(null);
const micStreamRef = useRef<MediaStream | null>(null);
const cameraStreamRef = useRef<MediaStream | null>(null);
const compositorRef = useRef<Compositor | null>(null);
const chunksRef = useRef<Blob[]>([]);
const startTimeRef = useRef<number>(0);
const cleanupIpcRef = useRef<(() => void) | null>(null);
@@ -150,18 +40,11 @@ export function useScreenRecorder({
onErrorRef.current = onError;
});
const modeRef = useRef(mode);
modeRef.current = mode;
const stopAllTracks = useCallback(() => {
compositorRef.current?.stop();
compositorRef.current = null;
screenStreamRef.current?.getTracks().forEach((t) => t.stop());
micStreamRef.current?.getTracks().forEach((t) => t.stop());
cameraStreamRef.current?.getTracks().forEach((t) => t.stop());
screenStreamRef.current = null;
micStreamRef.current = null;
cameraStreamRef.current = null;
recorderRef.current = null;
chunksRef.current = [];
cleanupIpcRef.current?.();
@@ -171,8 +54,6 @@ export function useScreenRecorder({
const startRecording = useCallback(
async (sourceId: string) => {
try {
const includeCamera = modeRef.current === "video";
// 1. Screen video
const screenStream = await navigator.mediaDevices.getUserMedia({
audio: false,
@@ -181,7 +62,7 @@ export function useScreenRecorder({
chromeMediaSource: "desktop",
chromeMediaSourceId: sourceId,
},
} as unknown as MediaTrackConstraints,
} as MediaTrackConstraints,
});
screenStreamRef.current = screenStream;
@@ -189,27 +70,9 @@ export function useScreenRecorder({
const micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
micStreamRef.current = micStream;
// 3. Camera (only in video mode)
let cameraStream: MediaStream | null = null;
if (includeCamera) {
try {
cameraStream = await navigator.mediaDevices.getUserMedia({
video: { aspectRatio: { ideal: 1 }, width: { ideal: 320 } },
audio: false,
});
cameraStreamRef.current = cameraStream;
} catch {
// Camera unavailable — proceed without it
}
}
// 4. Composite screen + camera via canvas
const compositor = createCompositor(screenStream, cameraStream);
compositorRef.current = compositor;
// 5. Combine composited video + mic audio
// 3. Combine screen video + mic audio
const combined = new MediaStream([
...compositor.stream.getVideoTracks(),
...screenStream.getVideoTracks(),
...micStream.getAudioTracks(),
]);
@@ -235,12 +98,12 @@ export function useScreenRecorder({
}
};
recorder.start();
recorder.start(1000);
// 6. Show floating control window
window.electronScreen.startRecordingWindow({ includeCamera });
// 4. Show floating control window
window.electronScreen.startRecordingWindow();
// 7. Listen for stop from floating window
// 5. Listen for stop from floating window
cleanupIpcRef.current = window.electronScreen.onStopRequested(() => {
if (recorderRef.current?.state === "recording") {
recorderRef.current.stop();