FlowyLabs/llink
1
0
Fork 0
Code Issues 71 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: allow recording with webcam overlay

Browse Source
This commit is contained in:
talksik
2026-04-08 16:46:31 -07:00
parent b2dbd65c19
commit 0bbddd1e60
8 changed files with 251 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/src/api/types.ts
+1
View File
@@ -121,6 +121,7 @@ export const MediaPropertiesSchema = z.object({
duration_ms: z.number(),
size_bytes: z.number(),
transcript: TranscriptSchema.optional(),
source: z.enum(["camera", "screen"]).optional(),
});
export type MediaProperties = z.infer<typeof MediaPropertiesSchema>;
js/src/electron.d.ts
Vendored
+2 -1
View File
@@ -32,12 +32,13 @@ declare global {
};
electronScreen: {
getScreenSources: () => Promise<ScreenSource[]>;
startRecordingWindow: () => void;
startRecordingWindow: (data: { includeCamera: boolean }) => void;
stopRecordingWindow: () => void;
onStopRequested: (callback: () => void) => () => void;
};
electronScreenRecord: {
stop: () => void;
onInit: (callback: (data: { includeCamera: boolean }) => void) => () => void;
};
electronLink: {
fetchMetadata: (url: string) => Promise<LinkMetadata | null>;
js/src/features/compose/compose-overlay.tsx
+5
View File
@@ -285,6 +285,7 @@ export function ComposeOverlay({
reviewMimeType,
);
const isAudioOnly = reviewMimeType.startsWith("audio/");
particleId = await createParticle.mutateAsync({
path,
type: "media",
@@ -293,6 +294,9 @@ export function ComposeOverlay({
mime_type: reviewMimeType,
duration_ms: reviewDurationMs,
size_bytes,
...(!isAudioOnly && {
source: recordingSource === "screen" ? "screen" as const : "camera" as const,
}),
},
createdByHumanId: userId,
});
@@ -309,6 +313,7 @@ export function ComposeOverlay({
reviewBlob,
reviewMimeType,
reviewDurationMs,
recordingSource,
createParticle,
uploadMedia,
uploadAttachments,
js/src/features/compose/use-screen-recorder.ts
+150 -11
View File
@@ -1,4 +1,5 @@
import { useCallback, useEffect, useRef } from "react";
import type { RecordingMode } from "@/stores/media-settings-store";
const VIDEO_PREFERRED_MIME = "video/webm;codecs=vp9,opus";
const VIDEO_FALLBACK_MIME = "video/webm";
@@ -9,23 +10,135 @@ function getScreenMime(): string {
: VIDEO_FALLBACK_MIME;
}
// ---------------------------------------------------------------------------
// Canvas compositor — overlays webcam as a circular PiP on the screen feed
// ---------------------------------------------------------------------------
interface Compositor {
/** Composited video stream (screen + optional webcam bubble). */
stream: MediaStream;
/** Tear down the animation loop and video elements. */
stop: () => void;
}
function createCompositor(
screenStream: MediaStream,
cameraStream: MediaStream | null,
): Compositor {
const screenTrack = screenStream.getVideoTracks()[0];
const settings = screenTrack.getSettings();
const width = settings.width ?? 1920;
const height = settings.height ?? 1080;
const canvas = document.createElement("canvas");
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext("2d")!;
// Hidden video elements used as frame sources
const screenVideo = document.createElement("video");
screenVideo.srcObject = screenStream;
screenVideo.muted = true;
screenVideo.playsInline = true;
screenVideo.play();
let cameraVideo: HTMLVideoElement | null = null;
if (cameraStream) {
cameraVideo = document.createElement("video");
cameraVideo.srcObject = cameraStream;
cameraVideo.muted = true;
cameraVideo.playsInline = true;
cameraVideo.play();
}
let animId = 0;
const draw = () => {
// Screen — full canvas
ctx.drawImage(screenVideo, 0, 0, width, height);
// Webcam — circular bubble in bottom-left
if (cameraVideo && cameraVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
const bubbleSize = Math.round(Math.min(width, height) * 0.18);
const margin = Math.round(bubbleSize * 0.3);
const cx = margin + bubbleSize / 2;
const cy = height - margin - bubbleSize / 2;
ctx.save();
ctx.beginPath();
ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2);
ctx.clip();
// Crop camera to square centre, mirror horizontally
const vw = cameraVideo.videoWidth || 1;
const vh = cameraVideo.videoHeight || 1;
const side = Math.min(vw, vh);
const sx = (vw - side) / 2;
const sy = (vh - side) / 2;
ctx.translate(cx, cy);
ctx.scale(-1, 1); // horizontal flip
ctx.drawImage(
cameraVideo,
sx, sy, side, side,
-bubbleSize / 2, -bubbleSize / 2, bubbleSize, bubbleSize,
);
ctx.restore();
// Subtle ring around the bubble
ctx.beginPath();
ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2);
ctx.strokeStyle = "rgba(255,255,255,0.25)";
ctx.lineWidth = 2;
ctx.stroke();
}
animId = requestAnimationFrame(draw);
};
draw();
return {
stream: canvas.captureStream(30),
stop: () => {
cancelAnimationFrame(animId);
screenVideo.pause();
screenVideo.srcObject = null;
if (cameraVideo) {
cameraVideo.pause();
cameraVideo.srcObject = null;
}
},
};
}
// ---------------------------------------------------------------------------
// Hook
// ---------------------------------------------------------------------------
interface UseScreenRecorderOptions {
/** "video" = screen + webcam overlay + mic. "audio" = screen + mic only. */
mode: RecordingMode;
onFinish: (blob: Blob, durationMs: number, mimeType: string) => void;
onError: (message: string) => void;
}
/**
* Manages screen recording via Electron's desktopCapturer.
* Captures screen video + mic audio into a single MediaRecorder.
* Communicates with the floating control window via IPC.
*
* When mode is "video", captures screen video composited with a webcam
* overlay (via Canvas) plus mic audio.
* When mode is "audio", captures screen video with mic audio only.
*/
export function useScreenRecorder({
mode,
onFinish,
onError,
}: UseScreenRecorderOptions) {
const recorderRef = useRef<MediaRecorder | null>(null);
const screenStreamRef = useRef<MediaStream | null>(null);
const micStreamRef = useRef<MediaStream | null>(null);
const cameraStreamRef = useRef<MediaStream | null>(null);
const compositorRef = useRef<Compositor | null>(null);
const chunksRef = useRef<Blob[]>([]);
const startTimeRef = useRef<number>(0);
const cleanupIpcRef = useRef<(() => void) | null>(null);
@@ -37,11 +150,18 @@ export function useScreenRecorder({
onErrorRef.current = onError;
});
const modeRef = useRef(mode);
modeRef.current = mode;
const stopAllTracks = useCallback(() => {
compositorRef.current?.stop();
compositorRef.current = null;
screenStreamRef.current?.getTracks().forEach((t) => t.stop());
micStreamRef.current?.getTracks().forEach((t) => t.stop());
cameraStreamRef.current?.getTracks().forEach((t) => t.stop());
screenStreamRef.current = null;
micStreamRef.current = null;
cameraStreamRef.current = null;
recorderRef.current = null;
chunksRef.current = [];
cleanupIpcRef.current?.();
@@ -51,7 +171,9 @@ export function useScreenRecorder({
const startRecording = useCallback(
async (sourceId: string) => {
try {
// Screen video via Electron desktopCapturer
const includeCamera = modeRef.current === "video";
// 1. Screen video
const screenStream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: {
@@ -63,14 +185,31 @@ export function useScreenRecorder({
});
screenStreamRef.current = screenStream;
// Mic audio separately
const micStream =
await navigator.mediaDevices.getUserMedia({ audio: true });
// 2. Mic audio
const micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
micStreamRef.current = micStream;
// Combine into a single stream
// 3. Camera (only in video mode)
let cameraStream: MediaStream | null = null;
if (includeCamera) {
try {
cameraStream = await navigator.mediaDevices.getUserMedia({
video: { aspectRatio: { ideal: 1 }, width: { ideal: 320 } },
audio: false,
});
cameraStreamRef.current = cameraStream;
} catch {
// Camera unavailable — proceed without it
}
}
// 4. Composite screen + camera via canvas
const compositor = createCompositor(screenStream, cameraStream);
compositorRef.current = compositor;
// 5. Combine composited video + mic audio
const combined = new MediaStream([
...screenStream.getVideoTracks(),
...compositor.stream.getVideoTracks(),
...micStream.getAudioTracks(),
]);
@@ -98,10 +237,10 @@ export function useScreenRecorder({
recorder.start();
// Show the floating control window
window.electronScreen.startRecordingWindow();
// 6. Show floating control window
window.electronScreen.startRecordingWindow({ includeCamera });
// Listen for stop from floating window
// 7. Listen for stop from floating window
cleanupIpcRef.current = window.electronScreen.onStopRequested(() => {
if (recorderRef.current?.state === "recording") {
recorderRef.current.stop();
js/src/features/particles/media-particle-view.tsx
+1 -1
View File
@@ -147,7 +147,7 @@ export const MediaParticleView = forwardRef<MediaParticleHandle, MediaParticleVi
playsInline
onEnded={onEnded}
onTimeUpdate={handleTimeUpdate}
className="h-full w-full object-cover"
className={`h-full w-full ${particle.properties.source === "screen" ? "object-contain bg-black" : "object-cover"}`}
/>
{transcript && (
js/src/main.ts
+23 -2
View File
@@ -209,9 +209,30 @@ ipcMain.handle('screen:get-sources', async () => {
});
// Screen recording IPC handlers
ipcMain.on('screen-record:start', () => {
ipcMain.on('screen-record:start', (_event, data: { includeCamera: boolean }) => {
createScreenRecordWindow();
screenRecordWindow?.showInactive();
if (!screenRecordWindow) return;
// Resize based on whether camera preview is shown
const winW = data.includeCamera ? 200 : 240;
const winH = data.includeCamera ? 176 : 48;
const { width, height } = screen.getPrimaryDisplay().workAreaSize;
screenRecordWindow.setSize(winW, winH);
screenRecordWindow.setPosition(
Math.round((width - winW) / 2),
height - winH - 32,
);
const send = () => {
screenRecordWindow?.webContents.send('screen-record:init', data);
screenRecordWindow?.showInactive();
};
if (screenRecordWindow.webContents.isLoading()) {
screenRecordWindow.webContents.once('did-finish-load', send);
} else {
send();
}
});
ipcMain.on('screen-record:stop', () => {
mainWindow?.webContents.send('screen-record:stopped');
js/src/preload.ts
+6 -1
View File
@@ -43,7 +43,7 @@ contextBridge.exposeInMainWorld('electronAutoplay', {
contextBridge.exposeInMainWorld('electronScreen', {
getScreenSources: () => ipcRenderer.invoke('screen:get-sources'),
startRecordingWindow: () => ipcRenderer.send('screen-record:start'),
startRecordingWindow: (data: { includeCamera: boolean }) => ipcRenderer.send('screen-record:start', data),
stopRecordingWindow: () => ipcRenderer.send('screen-record:cancel'),
onStopRequested: (callback: () => void) => {
const handler = () => callback();
@@ -54,6 +54,11 @@ contextBridge.exposeInMainWorld('electronScreen', {
contextBridge.exposeInMainWorld('electronScreenRecord', {
stop: () => ipcRenderer.send('screen-record:stop'),
onInit: (callback: (data: { includeCamera: boolean }) => void) => {
const handler = (_event: Electron.IpcRendererEvent, data: { includeCamera: boolean }) => callback(data);
ipcRenderer.on('screen-record:init', handler);
return () => { ipcRenderer.removeListener('screen-record:init', handler); };
},
});
contextBridge.exposeInMainWorld('electronLink', {
js/src/screen_record_window/ScreenRecordControlApp.tsx
+63 -4
View File
@@ -1,9 +1,48 @@
import { useEffect, useState } from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { Square } from "lucide-react";
export function ScreenRecordControlApp() {
const [elapsed, setElapsed] = useState(0);
const [includeCamera, setIncludeCamera] = useState(false);
const videoRef = useRef<HTMLVideoElement>(null);
const streamRef = useRef<MediaStream | null>(null);
// Listen for init data from main process
useEffect(() => {
return window.electronScreenRecord.onInit((data) => {
setIncludeCamera(data.includeCamera);
});
}, []);
// Acquire webcam for preview (independent from the recording capture)
useEffect(() => {
if (!includeCamera) return;
let cancelled = false;
navigator.mediaDevices
.getUserMedia({ video: { aspectRatio: { ideal: 1 }, width: { ideal: 160 } }, audio: false })
.then((stream) => {
if (cancelled) {
stream.getTracks().forEach((t) => t.stop());
return;
}
streamRef.current = stream;
if (videoRef.current) {
videoRef.current.srcObject = stream;
}
})
.catch(() => {
// Camera unavailable — just don't show preview
});
return () => {
cancelled = true;
streamRef.current?.getTracks().forEach((t) => t.stop());
streamRef.current = null;
};
}, [includeCamera]);
// Timer
useEffect(() => {
const interval = setInterval(() => setElapsed((prev) => prev + 1), 1000);
return () => clearInterval(interval);
@@ -13,13 +52,33 @@ export function ScreenRecordControlApp() {
const seconds = elapsed % 60;
const display = `${minutes}:${seconds.toString().padStart(2, "0")}`;
const handleStop = useCallback(() => {
streamRef.current?.getTracks().forEach((t) => t.stop());
streamRef.current = null;
window.electronScreenRecord.stop();
}, []);
return (
<div className="flex h-screen w-screen items-center justify-center bg-zinc-900">
<div className="flex items-center gap-3 px-4">
<div className="flex h-screen w-screen flex-col items-center justify-center gap-2 bg-zinc-900 px-4 py-3">
{/* Webcam preview */}
{includeCamera && (
<div className="relative h-24 w-24 overflow-hidden rounded-full bg-zinc-800">
<video
ref={videoRef}
autoPlay
playsInline
muted
className="h-full w-full -scale-x-100 object-cover"
/>
</div>
)}
{/* Controls */}
<div className="flex items-center gap-3">
<span className="h-2.5 w-2.5 animate-pulse rounded-full bg-red-500" />
<span className="font-mono text-sm text-white/80">{display}</span>
<button
onClick={() => window.electronScreenRecord.stop()}
onClick={handleStop}
className="flex items-center gap-1.5 rounded-md bg-red-600 px-3 py-1 text-xs font-medium text-white transition-colors hover:bg-red-500"
>
<Square className="size-3 fill-current" />