From bebd814c4604f2d5a423a3d952e08af07b88aef6 Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 8 Apr 2026 11:25:21 -0700 Subject: [PATCH 01/10] chore: bump version --- js/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/package.json b/js/package.json index 4dd5c9c..a3386c6 100644 --- a/js/package.json +++ b/js/package.json @@ -1,7 +1,7 @@ { "name": "Flowy.llink", "productName": "Flowy.llink", - "version": "0.7.0", + "version": "0.8.0", "description": "Flowy.llink is a team communication app for teams", "main": ".vite/build/main.js", "private": true, -- 2.54.0 From 36fbb2acfd594e3e8e8791d3b485f43cafd0e05e Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 8 Apr 2026 16:04:55 -0700 Subject: [PATCH 02/10] first pass implementation --- js/forge.config.ts | 4 + js/src/electron.d.ts | 9 ++ js/src/features/compose/compose-overlay.tsx | 89 ++++++++++- .../features/compose/screen-source-picker.tsx | 132 ++++++++++++++++ .../features/compose/use-screen-recorder.ts | 149 ++++++++++++++++++ js/src/features/particles/stream-view.tsx | 7 + js/src/main.ts | 52 ++++++ js/src/preload.ts | 15 ++ .../ScreenRecordControlApp.tsx | 31 ++++ js/src/screen_record_window/index.html | 11 ++ js/src/screen_record_window/renderer.tsx | 6 + js/src/vite-env.d.ts | 2 + js/vite.screen-record.config.mts | 23 +++ 13 files changed, 522 insertions(+), 8 deletions(-) create mode 100644 js/src/features/compose/screen-source-picker.tsx create mode 100644 js/src/features/compose/use-screen-recorder.ts create mode 100644 js/src/screen_record_window/ScreenRecordControlApp.tsx create mode 100644 js/src/screen_record_window/index.html create mode 100644 js/src/screen_record_window/renderer.tsx create mode 100644 js/vite.screen-record.config.mts diff --git a/js/forge.config.ts b/js/forge.config.ts index e8dcd90..56fb53e 100644 --- a/js/forge.config.ts +++ b/js/forge.config.ts @@ -76,6 +76,10 @@ const config: ForgeConfig = { name: 'huddle_window', config: 'vite.huddle.config.mts', }, + { + name: 'screen_record_window', + config: 'vite.screen-record.config.mts', + }, ], }), // Fuses are used to enable/disable various Electron functionality diff --git a/js/src/electron.d.ts b/js/src/electron.d.ts index 343deaf..e8cc949 100644 --- a/js/src/electron.d.ts +++ b/js/src/electron.d.ts @@ -30,6 +30,15 @@ declare global { onStop: (callback: () => void) => () => void; onNavigate: (callback: (data: { networkId: string; streamId: string }) => void) => () => void; }; + electronScreen: { + getScreenSources: () => Promise; + startRecordingWindow: () => void; + stopRecordingWindow: () => void; + onStopRequested: (callback: () => void) => () => void; + }; + electronScreenRecord: { + stop: () => void; + }; electronLink: { fetchMetadata: (url: string) => Promise; openExternal: (url: string) => Promise; diff --git a/js/src/features/compose/compose-overlay.tsx b/js/src/features/compose/compose-overlay.tsx index dedc065..0074ad4 100644 --- a/js/src/features/compose/compose-overlay.tsx +++ b/js/src/features/compose/compose-overlay.tsx @@ -3,9 +3,11 @@ import { toast } from "sonner"; import { useAuthStore } from "@/stores/auth-store"; import { useCreateParticle, useCreateStreamParticle } from "@/hooks/use-create-particle"; import { useRecorder } from "@/features/compose/use-recorder"; +import { useScreenRecorder } from "@/features/compose/use-screen-recorder"; import { particlePath, parseParticlePath } from "@/lib/particle-path"; import type { ParticlePath } from "@/lib/particle-path"; import { RecordingOverlay } from "@/features/compose/recording-overlay"; +import { ScreenSourcePicker } from "@/features/compose/screen-source-picker"; import { TextComposeStep } from "@/features/compose/text-compose-step"; import { ConfigureStreamStep } from "@/features/compose/configure-stream-step"; import { apiClient } from "@/api/client"; @@ -15,7 +17,9 @@ import { createImageThumbnail } from "@/lib/image-thumbnail"; import { MAX_ATTACHMENT_SIZE_BYTES, MAX_ATTACHMENTS } from "@/lib/constants"; import type { PendingAttachment } from "@/features/compose/attachment-strip"; -type ComposeStep = "idle" | "recording" | "reviewing" | "typing" | "configuring" | "submitting"; +type ComposeStep = "idle" | "picking" | "recording" | "reviewing" | "typing" | "configuring" | "submitting"; + +type RecordingSource = "media" | "screen"; interface ComposeOverlayProps { networkId: string; @@ -50,6 +54,7 @@ export function ComposeOverlay({ const [reviewMimeType, setReviewMimeType] = useState(null); const [attachments, setAttachments] = useState([]); + const [recordingSource, setRecordingSource] = useState("media"); const recordingMode = useMediaSettingsStore((s) => s.recordingMode); const userId = useAuthStore((s) => s.user?.id); @@ -61,6 +66,8 @@ export function ComposeOverlay({ const recordStartRef = useRef(0); const disabledRef = useRef(disabled); disabledRef.current = disabled; + const recordingSourceRef = useRef(recordingSource); + recordingSourceRef.current = recordingSource; const setStepSync = useCallback((next: ComposeStep) => { stepRef.current = next; @@ -86,6 +93,7 @@ export function ComposeOverlay({ setReviewBlob(null); setReviewDurationMs(0); setReviewMimeType(null); + setRecordingSource("media"); setAttachments((prev) => { revokeAttachmentThumbnails(prev); return []; @@ -151,6 +159,23 @@ export function ComposeOverlay({ onError: (message) => setError(message), }); + const { + startRecording: startScreenRecording, + stopRecording: stopScreenRecording, + cancelRecording: cancelScreenRecording, + } = useScreenRecorder({ + onFinish: (blob, durationMs, mimeType) => { + setStepSync("reviewing"); + setReviewBlob(blob); + setReviewDurationMs(durationMs); + setReviewMimeType(mimeType); + }, + onError: (message) => { + setError(message); + cancel(); + }, + }); + // --- Submission --- const uploadMedia = useCallback( @@ -327,7 +352,7 @@ export function ComposeOverlay({ const handleKeyDown = (e: KeyboardEvent) => { const currentStep = stepRef.current; - if (currentStep === "typing" || currentStep === "configuring") { + if (currentStep === "typing" || currentStep === "configuring" || currentStep === "picking") { if (e.key === "Escape") { e.preventDefault(); cancel(); @@ -347,7 +372,7 @@ export function ComposeOverlay({ switch (currentStep) { case "idle": { if (disabledRef.current) { - if ((e.key === "`" && !e.repeat) || e.key === "t" || e.key === "T") { + if ((e.key === "`" && !e.repeat) || e.key === "t" || e.key === "T" || e.key === "s" || e.key === "S") { e.preventDefault(); toast.info("This stream is closed"); } @@ -356,8 +381,13 @@ export function ComposeOverlay({ if (e.key === "`" && !e.repeat) { e.preventDefault(); recordStartRef.current = Date.now(); + setRecordingSource("media"); setStepSync("recording"); startRecording(); + } else if (e.key === "s" || e.key === "S") { + e.preventDefault(); + setRecordingSource("screen"); + setStepSync("picking"); } else if (e.key === "t" || e.key === "T") { e.preventDefault(); setStepSync("typing"); @@ -370,9 +400,17 @@ export function ComposeOverlay({ // Second tap stops recording (toggle mode) e.preventDefault(); stopRecording(); + } else if ((e.key === "s" || e.key === "S") && recordingSourceRef.current === "screen") { + // S stops screen recording when main window is focused + e.preventDefault(); + stopScreenRecording(); } else if (e.key === "q" || e.key === "Q" || e.key === "Escape") { e.preventDefault(); - cancelRecording(); + if (recordingSourceRef.current === "screen") { + cancelScreenRecording(); + } else { + cancelRecording(); + } cancel(); } break; @@ -381,7 +419,11 @@ export function ComposeOverlay({ case "reviewing": { if (e.key === "q" || e.key === "Q" || e.key === "Escape") { e.preventDefault(); - cancelRecording(); + if (recordingSourceRef.current === "screen") { + cancelScreenRecording(); + } else { + cancelRecording(); + } cancel(); } else if (e.key === "Enter") { e.preventDefault(); @@ -397,7 +439,7 @@ export function ComposeOverlay({ }; const handleKeyUp = (e: KeyboardEvent) => { - if (stepRef.current === "recording" && e.key === "`") { + if (stepRef.current === "recording" && e.key === "`" && recordingSourceRef.current === "media") { e.preventDefault(); // Only stop on release if held long enough (hold-to-record mode). // Quick taps are handled by the second keydown (toggle mode). @@ -413,7 +455,17 @@ export function ComposeOverlay({ window.removeEventListener("keydown", handleKeyDown); window.removeEventListener("keyup", handleKeyUp); }; - }, [targetPath, startRecording, stopRecording, cancelRecording, cancel, setStepSync]); + }, [targetPath, startRecording, stopRecording, cancelRecording, startScreenRecording, stopScreenRecording, cancelScreenRecording, cancel, setStepSync]); + + // --- Screen source selection handler --- + + const handleScreenSourceSelected = useCallback( + (sourceId: string) => { + setStepSync("recording"); + startScreenRecording(sourceId); + }, + [setStepSync, startScreenRecording], + ); // --- Render --- @@ -425,7 +477,13 @@ export function ComposeOverlay({ return ( <> - {(step === "recording" || step === "reviewing") && ( + {step === "picking" && ( + + )} + {(step === "recording" || step === "reviewing") && recordingSource === "media" && ( )} + {step === "reviewing" && recordingSource === "screen" && reviewBlob && ( + + )} {step === "typing" && ( void; + onCancel: () => void; +} + +export function ScreenSourcePicker({ onSelect, onCancel }: ScreenSourcePickerProps) { + const [sources, setSources] = useState([]); + const [selectedId, setSelectedId] = useState(null); + const [loading, setLoading] = useState(true); + + useEffect(() => { + window.electronScreen.getScreenSources().then((result) => { + setSources(result); + setLoading(false); + }); + }, []); + + // Auto-select if there's only one screen and no windows + useEffect(() => { + if (!loading && sources.length === 1) { + setSelectedId(sources[0].id); + } + }, [loading, sources]); + + const screens = sources.filter((s) => s.id.startsWith("screen:")); + const windows = sources.filter((s) => s.id.startsWith("window:")); + + return ( +
+
+
+

+ Record your screen +

+ +
+ +
+ {loading ? ( +

+ Loading sources... +

+ ) : ( + <> + {screens.length > 0 && ( + + )} + {windows.length > 0 && ( + + )} + + )} +
+ +
+ + +
+
+
+ ); +} + +function SourceSection({ + title, + sources, + selectedId, + onSelect, +}: { + title: string; + sources: ScreenSource[]; + selectedId: string | null; + onSelect: (id: string) => void; +}) { + return ( +
+

+ {title} +

+
+ {sources.map((source) => ( + + ))} +
+
+ ); +} diff --git a/js/src/features/compose/use-screen-recorder.ts b/js/src/features/compose/use-screen-recorder.ts new file mode 100644 index 0000000..52b1de1 --- /dev/null +++ b/js/src/features/compose/use-screen-recorder.ts @@ -0,0 +1,149 @@ +import { useCallback, useEffect, useRef } from "react"; + +const VIDEO_PREFERRED_MIME = "video/webm;codecs=vp9,opus"; +const VIDEO_FALLBACK_MIME = "video/webm"; + +function getScreenMime(): string { + return MediaRecorder.isTypeSupported(VIDEO_PREFERRED_MIME) + ? VIDEO_PREFERRED_MIME + : VIDEO_FALLBACK_MIME; +} + +interface UseScreenRecorderOptions { + onFinish: (blob: Blob, durationMs: number, mimeType: string) => void; + onError: (message: string) => void; +} + +/** + * Manages screen recording via Electron's desktopCapturer. + * Captures screen video + mic audio into a single MediaRecorder. + * Communicates with the floating control window via IPC. + */ +export function useScreenRecorder({ + onFinish, + onError, +}: UseScreenRecorderOptions) { + const recorderRef = useRef(null); + const screenStreamRef = useRef(null); + const micStreamRef = useRef(null); + const chunksRef = useRef([]); + const startTimeRef = useRef(0); + const cleanupIpcRef = useRef<(() => void) | null>(null); + + const onFinishRef = useRef(onFinish); + const onErrorRef = useRef(onError); + useEffect(() => { + onFinishRef.current = onFinish; + onErrorRef.current = onError; + }); + + const stopAllTracks = useCallback(() => { + screenStreamRef.current?.getTracks().forEach((t) => t.stop()); + micStreamRef.current?.getTracks().forEach((t) => t.stop()); + screenStreamRef.current = null; + micStreamRef.current = null; + recorderRef.current = null; + chunksRef.current = []; + cleanupIpcRef.current?.(); + cleanupIpcRef.current = null; + }, []); + + const startRecording = useCallback( + async (sourceId: string) => { + try { + // Screen video via Electron desktopCapturer + const screenStream = await navigator.mediaDevices.getUserMedia({ + audio: false, + video: { + mandatory: { + chromeMediaSource: "desktop", + chromeMediaSourceId: sourceId, + }, + } as unknown as MediaTrackConstraints, + }); + screenStreamRef.current = screenStream; + + // Mic audio separately + const micStream = + await navigator.mediaDevices.getUserMedia({ audio: true }); + micStreamRef.current = micStream; + + // Combine into a single stream + const combined = new MediaStream([ + ...screenStream.getVideoTracks(), + ...micStream.getAudioTracks(), + ]); + + chunksRef.current = []; + startTimeRef.current = Date.now(); + + const mime = getScreenMime(); + const recorder = new MediaRecorder(combined, { mimeType: mime }); + recorderRef.current = recorder; + + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunksRef.current.push(e.data); + }; + + recorder.onstop = () => { + const durationMs = Date.now() - startTimeRef.current; + const blob = new Blob(chunksRef.current, { type: mime }); + stopAllTracks(); + window.electronScreen.stopRecordingWindow(); + + if (blob.size > 0) { + onFinishRef.current(blob, durationMs, mime); + } + }; + + recorder.start(); + + // Show the floating control window + window.electronScreen.startRecordingWindow(); + + // Listen for stop from floating window + const cleanup = window.electronScreen.onStopRequested(() => { + if (recorderRef.current?.state === "recording") { + recorderRef.current.stop(); + } + }); + cleanupIpcRef.current = cleanup; + } catch (err) { + stopAllTracks(); + window.electronScreen.stopRecordingWindow(); + onErrorRef.current( + err instanceof Error ? err.message : "Failed to start screen recording", + ); + } + }, + [stopAllTracks], + ); + + const stopRecording = useCallback(() => { + if (recorderRef.current?.state === "recording") { + recorderRef.current.stop(); + } + }, []); + + const cancelRecording = useCallback(() => { + if (recorderRef.current) { + recorderRef.current.ondataavailable = null; + recorderRef.current.onstop = null; + if (recorderRef.current.state === "recording") { + recorderRef.current.stop(); + } + } + stopAllTracks(); + window.electronScreen.stopRecordingWindow(); + }, [stopAllTracks]); + + // Cleanup on unmount + useEffect(() => { + return () => { + stopAllTracks(); + window.electronScreen.stopRecordingWindow(); + }; + }, [stopAllTracks]); + + return { startRecording, stopRecording, cancelRecording }; +} diff --git a/js/src/features/particles/stream-view.tsx b/js/src/features/particles/stream-view.tsx index 31a1f2b..becbe1f 100644 --- a/js/src/features/particles/stream-view.tsx +++ b/js/src/features/particles/stream-view.tsx @@ -130,6 +130,7 @@ const STREAM_VIEW_KEYBINDINGS: KeybindingGroup[] = [ label: "Compose", bindings: [ { keys: ["Hold", "`"], description: "Reply" }, + { keys: ["S"], description: "Screen record" }, { keys: ["T"], description: "Text compose" }, { keys: ["H"], description: "Join huddle" }, ], @@ -541,6 +542,12 @@ function StreamViewControls({ {" "} to reply + + + S + {" "} + screen + T diff --git a/js/src/main.ts b/js/src/main.ts index 2aeda3c..249824d 100644 --- a/js/src/main.ts +++ b/js/src/main.ts @@ -25,6 +25,7 @@ if (process.platform === 'darwin' && !app.isPackaged) { let mainWindow: BrowserWindow | null = null; let autoplayWindow: BrowserWindow | null = null; let huddleWindow: BrowserWindow | null = null; +let screenRecordWindow: BrowserWindow | null = null; const createWindow = () => { mainWindow = new BrowserWindow({ @@ -121,6 +122,43 @@ function positionAutoplayWindow() { autoplayWindow.setPosition(width - winW - 16, 16); } +const createScreenRecordWindow = () => { + if (screenRecordWindow) return; + + const { width, height } = screen.getPrimaryDisplay().workAreaSize; + const winW = 240; + const winH = 48; + + screenRecordWindow = new BrowserWindow({ + width: winW, + height: winH, + x: Math.round((width - winW) / 2), + y: height - winH - 32, + resizable: false, + frame: false, + alwaysOnTop: true, + skipTaskbar: true, + focusable: true, + show: false, + webPreferences: { + preload: path.join(__dirname, 'preload.js'), + }, + }); + screenRecordWindow.setVisibleOnAllWorkspaces(true); + + if (SCREEN_RECORD_WINDOW_VITE_DEV_SERVER_URL) { + screenRecordWindow.loadURL(SCREEN_RECORD_WINDOW_VITE_DEV_SERVER_URL); + } else { + screenRecordWindow.loadFile( + path.join(__dirname, `../renderer/${SCREEN_RECORD_WINDOW_VITE_NAME}/index.html`), + ); + } + + screenRecordWindow.on('closed', () => { + screenRecordWindow = null; + }); +}; + // Window control IPC handlers ipcMain.on('window:minimize', (event) => { BrowserWindow.fromWebContents(event.sender)?.minimize(); @@ -170,6 +208,20 @@ ipcMain.handle('screen:get-sources', async () => { })); }); +// Screen recording IPC handlers +ipcMain.on('screen-record:start', () => { + createScreenRecordWindow(); + screenRecordWindow?.showInactive(); +}); +ipcMain.on('screen-record:stop', () => { + mainWindow?.webContents.send('screen-record:stopped'); + screenRecordWindow?.close(); + mainWindow?.focus(); +}); +ipcMain.on('screen-record:cancel', () => { + screenRecordWindow?.close(); +}); + // Autoplay IPC handlers ipcMain.on('autoplay:play', (_event, payload) => { if (!autoplayWindow) createAutoplayWindow(); diff --git a/js/src/preload.ts b/js/src/preload.ts index d6a20b3..ef5f076 100644 --- a/js/src/preload.ts +++ b/js/src/preload.ts @@ -41,6 +41,21 @@ contextBridge.exposeInMainWorld('electronAutoplay', { }, }); +contextBridge.exposeInMainWorld('electronScreen', { + getScreenSources: () => ipcRenderer.invoke('screen:get-sources'), + startRecordingWindow: () => ipcRenderer.send('screen-record:start'), + stopRecordingWindow: () => ipcRenderer.send('screen-record:cancel'), + onStopRequested: (callback: () => void) => { + const handler = () => callback(); + ipcRenderer.on('screen-record:stopped', handler); + return () => { ipcRenderer.removeListener('screen-record:stopped', handler); }; + }, +}); + +contextBridge.exposeInMainWorld('electronScreenRecord', { + stop: () => ipcRenderer.send('screen-record:stop'), +}); + contextBridge.exposeInMainWorld('electronLink', { fetchMetadata: (url: string) => ipcRenderer.invoke('link:fetch-metadata', url), openExternal: (url: string) => ipcRenderer.invoke('link:open-external', url), diff --git a/js/src/screen_record_window/ScreenRecordControlApp.tsx b/js/src/screen_record_window/ScreenRecordControlApp.tsx new file mode 100644 index 0000000..41cac98 --- /dev/null +++ b/js/src/screen_record_window/ScreenRecordControlApp.tsx @@ -0,0 +1,31 @@ +import { useEffect, useState } from "react"; +import { Square } from "lucide-react"; + +export function ScreenRecordControlApp() { + const [elapsed, setElapsed] = useState(0); + + useEffect(() => { + const interval = setInterval(() => setElapsed((prev) => prev + 1), 1000); + return () => clearInterval(interval); + }, []); + + const minutes = Math.floor(elapsed / 60); + const seconds = elapsed % 60; + const display = `${minutes}:${seconds.toString().padStart(2, "0")}`; + + return ( +
+
+ + {display} + +
+
+ ); +} diff --git a/js/src/screen_record_window/index.html b/js/src/screen_record_window/index.html new file mode 100644 index 0000000..d046e78 --- /dev/null +++ b/js/src/screen_record_window/index.html @@ -0,0 +1,11 @@ + + + + + llink - Recording + + +
+ + + diff --git a/js/src/screen_record_window/renderer.tsx b/js/src/screen_record_window/renderer.tsx new file mode 100644 index 0000000..69357e2 --- /dev/null +++ b/js/src/screen_record_window/renderer.tsx @@ -0,0 +1,6 @@ +import { createRoot } from 'react-dom/client'; +import { ScreenRecordControlApp } from './ScreenRecordControlApp'; +import '@/styles/globals.css'; + +const root = createRoot(document.getElementById('root')!); +root.render(); diff --git a/js/src/vite-env.d.ts b/js/src/vite-env.d.ts index 6c35629..5f22f57 100644 --- a/js/src/vite-env.d.ts +++ b/js/src/vite-env.d.ts @@ -6,3 +6,5 @@ declare const AUTOPLAY_WINDOW_VITE_DEV_SERVER_URL: string | undefined; declare const AUTOPLAY_WINDOW_VITE_NAME: string; declare const HUDDLE_WINDOW_VITE_DEV_SERVER_URL: string | undefined; declare const HUDDLE_WINDOW_VITE_NAME: string; +declare const SCREEN_RECORD_WINDOW_VITE_DEV_SERVER_URL: string | undefined; +declare const SCREEN_RECORD_WINDOW_VITE_NAME: string; diff --git a/js/vite.screen-record.config.mts b/js/vite.screen-record.config.mts new file mode 100644 index 0000000..34c2e62 --- /dev/null +++ b/js/vite.screen-record.config.mts @@ -0,0 +1,23 @@ +import path from "path" +import tailwindcss from "@tailwindcss/vite" +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; + +// https://vitejs.dev/config +export default defineConfig({ + root: path.resolve(__dirname, './src/screen_record_window'), + plugins: [react(), tailwindcss()], + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, + build: { + outDir: path.resolve(__dirname, '.vite/renderer/screen_record_window'), + }, + server: { + fs: { + allow: [path.resolve(__dirname)], + }, + }, +}); -- 2.54.0 From ca7b408d37d409f116b1fcbd3e71e4a5f2cee3f3 Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 8 Apr 2026 16:07:13 -0700 Subject: [PATCH 03/10] remove screenrecord shortcut hint --- js/src/features/particles/stream-view.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/js/src/features/particles/stream-view.tsx b/js/src/features/particles/stream-view.tsx index becbe1f..e2db8a1 100644 --- a/js/src/features/particles/stream-view.tsx +++ b/js/src/features/particles/stream-view.tsx @@ -542,12 +542,6 @@ function StreamViewControls({ {" "} to reply - - - S - {" "} - screen - T -- 2.54.0 From 4c6095765796ae72f1f7848bfc874cc42466f800 Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 8 Apr 2026 16:13:30 -0700 Subject: [PATCH 04/10] refactor --- js/src/features/compose/use-screen-recorder.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/js/src/features/compose/use-screen-recorder.ts b/js/src/features/compose/use-screen-recorder.ts index 52b1de1..40f4a1a 100644 --- a/js/src/features/compose/use-screen-recorder.ts +++ b/js/src/features/compose/use-screen-recorder.ts @@ -102,12 +102,11 @@ export function useScreenRecorder({ window.electronScreen.startRecordingWindow(); // Listen for stop from floating window - const cleanup = window.electronScreen.onStopRequested(() => { + cleanupIpcRef.current = window.electronScreen.onStopRequested(() => { if (recorderRef.current?.state === "recording") { recorderRef.current.stop(); } }); - cleanupIpcRef.current = cleanup; } catch (err) { stopAllTracks(); window.electronScreen.stopRecordingWindow(); -- 2.54.0 From b2dbd65c197c09b780909fec2654a66d7c3908b3 Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 8 Apr 2026 16:38:10 -0700 Subject: [PATCH 05/10] fix: prevent mirror review of screen recording --- js/src/features/compose/compose-overlay.tsx | 3 +++ js/src/features/compose/recording-overlay.tsx | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/js/src/features/compose/compose-overlay.tsx b/js/src/features/compose/compose-overlay.tsx index 0074ad4..140f46c 100644 --- a/js/src/features/compose/compose-overlay.tsx +++ b/js/src/features/compose/compose-overlay.tsx @@ -164,6 +164,7 @@ export function ComposeOverlay({ stopRecording: stopScreenRecording, cancelRecording: cancelScreenRecording, } = useScreenRecorder({ + mode: recordingMode, onFinish: (blob, durationMs, mimeType) => { setStepSync("reviewing"); setReviewBlob(blob); @@ -496,6 +497,7 @@ export function ComposeOverlay({ onAddFiles={openFilePicker} isDragging={isDragging} dropZoneProps={dropZoneProps} + mirror={true} /> )} {step === "reviewing" && recordingSource === "screen" && reviewBlob && ( @@ -511,6 +513,7 @@ export function ComposeOverlay({ onAddFiles={openFilePicker} isDragging={isDragging} dropZoneProps={dropZoneProps} + mirror={false} /> )} {step === "typing" && ( diff --git a/js/src/features/compose/recording-overlay.tsx b/js/src/features/compose/recording-overlay.tsx index 8615622..914ecae 100644 --- a/js/src/features/compose/recording-overlay.tsx +++ b/js/src/features/compose/recording-overlay.tsx @@ -25,6 +25,8 @@ interface RecordingOverlayProps { onDragLeave: (e: React.DragEvent) => void; onDrop: (e: React.DragEvent) => void; }; + /** Mirror the video horizontally. Defaults to true (selfie-view for webcam). Set false for screen recordings. */ + mirror?: boolean; } function RecordingTimer() { @@ -52,9 +54,11 @@ function RecordingTimer() { function ReviewPlayback({ blob, isVideo, + mirror = true, }: { blob: Blob; isVideo: boolean; + mirror?: boolean; }) { const urlRef = useRef(null); const [objectUrl, setObjectUrl] = useState(null); @@ -82,7 +86,7 @@ function ReviewPlayback({ autoPlay loop playsInline - className="absolute inset-0 h-full w-full -scale-x-100 object-cover" + className={`absolute inset-0 h-full w-full object-cover${mirror ? " -scale-x-100" : ""}`} /> ); } @@ -119,6 +123,7 @@ export function RecordingOverlay({ onAddFiles, isDragging, dropZoneProps, + mirror = true, }: RecordingOverlayProps) { const videoRef = useRef(null); const recordingAudioSource = useAudioSource(mediaStream ?? null); @@ -176,6 +181,7 @@ export function RecordingOverlay({ )} -- 2.54.0 From 0bbddd1e60a03fd593b1c443e993509011ff2d14 Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 8 Apr 2026 16:46:31 -0700 Subject: [PATCH 06/10] feat: allow recording with webcam overlay --- js/src/api/types.ts | 1 + js/src/electron.d.ts | 3 +- js/src/features/compose/compose-overlay.tsx | 5 + .../features/compose/use-screen-recorder.ts | 161 ++++++++++++++++-- .../particles/media-particle-view.tsx | 2 +- js/src/main.ts | 25 ++- js/src/preload.ts | 7 +- .../ScreenRecordControlApp.tsx | 67 +++++++- 8 files changed, 251 insertions(+), 20 deletions(-) diff --git a/js/src/api/types.ts b/js/src/api/types.ts index 2e99ee8..3f4fde1 100644 --- a/js/src/api/types.ts +++ b/js/src/api/types.ts @@ -121,6 +121,7 @@ export const MediaPropertiesSchema = z.object({ duration_ms: z.number(), size_bytes: z.number(), transcript: TranscriptSchema.optional(), + source: z.enum(["camera", "screen"]).optional(), }); export type MediaProperties = z.infer; diff --git a/js/src/electron.d.ts b/js/src/electron.d.ts index e8cc949..1c11cdd 100644 --- a/js/src/electron.d.ts +++ b/js/src/electron.d.ts @@ -32,12 +32,13 @@ declare global { }; electronScreen: { getScreenSources: () => Promise; - startRecordingWindow: () => void; + startRecordingWindow: (data: { includeCamera: boolean }) => void; stopRecordingWindow: () => void; onStopRequested: (callback: () => void) => () => void; }; electronScreenRecord: { stop: () => void; + onInit: (callback: (data: { includeCamera: boolean }) => void) => () => void; }; electronLink: { fetchMetadata: (url: string) => Promise; diff --git a/js/src/features/compose/compose-overlay.tsx b/js/src/features/compose/compose-overlay.tsx index 140f46c..39331b4 100644 --- a/js/src/features/compose/compose-overlay.tsx +++ b/js/src/features/compose/compose-overlay.tsx @@ -285,6 +285,7 @@ export function ComposeOverlay({ reviewMimeType, ); + const isAudioOnly = reviewMimeType.startsWith("audio/"); particleId = await createParticle.mutateAsync({ path, type: "media", @@ -293,6 +294,9 @@ export function ComposeOverlay({ mime_type: reviewMimeType, duration_ms: reviewDurationMs, size_bytes, + ...(!isAudioOnly && { + source: recordingSource === "screen" ? "screen" as const : "camera" as const, + }), }, createdByHumanId: userId, }); @@ -309,6 +313,7 @@ export function ComposeOverlay({ reviewBlob, reviewMimeType, reviewDurationMs, + recordingSource, createParticle, uploadMedia, uploadAttachments, diff --git a/js/src/features/compose/use-screen-recorder.ts b/js/src/features/compose/use-screen-recorder.ts index 40f4a1a..82bc296 100644 --- a/js/src/features/compose/use-screen-recorder.ts +++ b/js/src/features/compose/use-screen-recorder.ts @@ -1,4 +1,5 @@ import { useCallback, useEffect, useRef } from "react"; +import type { RecordingMode } from "@/stores/media-settings-store"; const VIDEO_PREFERRED_MIME = "video/webm;codecs=vp9,opus"; const VIDEO_FALLBACK_MIME = "video/webm"; @@ -9,23 +10,135 @@ function getScreenMime(): string { : VIDEO_FALLBACK_MIME; } +// --------------------------------------------------------------------------- +// Canvas compositor — overlays webcam as a circular PiP on the screen feed +// --------------------------------------------------------------------------- + +interface Compositor { + /** Composited video stream (screen + optional webcam bubble). */ + stream: MediaStream; + /** Tear down the animation loop and video elements. */ + stop: () => void; +} + +function createCompositor( + screenStream: MediaStream, + cameraStream: MediaStream | null, +): Compositor { + const screenTrack = screenStream.getVideoTracks()[0]; + const settings = screenTrack.getSettings(); + const width = settings.width ?? 1920; + const height = settings.height ?? 1080; + + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext("2d")!; + + // Hidden video elements used as frame sources + const screenVideo = document.createElement("video"); + screenVideo.srcObject = screenStream; + screenVideo.muted = true; + screenVideo.playsInline = true; + screenVideo.play(); + + let cameraVideo: HTMLVideoElement | null = null; + if (cameraStream) { + cameraVideo = document.createElement("video"); + cameraVideo.srcObject = cameraStream; + cameraVideo.muted = true; + cameraVideo.playsInline = true; + cameraVideo.play(); + } + + let animId = 0; + + const draw = () => { + // Screen — full canvas + ctx.drawImage(screenVideo, 0, 0, width, height); + + // Webcam — circular bubble in bottom-left + if (cameraVideo && cameraVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) { + const bubbleSize = Math.round(Math.min(width, height) * 0.18); + const margin = Math.round(bubbleSize * 0.3); + const cx = margin + bubbleSize / 2; + const cy = height - margin - bubbleSize / 2; + + ctx.save(); + ctx.beginPath(); + ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2); + ctx.clip(); + + // Crop camera to square centre, mirror horizontally + const vw = cameraVideo.videoWidth || 1; + const vh = cameraVideo.videoHeight || 1; + const side = Math.min(vw, vh); + const sx = (vw - side) / 2; + const sy = (vh - side) / 2; + + ctx.translate(cx, cy); + ctx.scale(-1, 1); // horizontal flip + ctx.drawImage( + cameraVideo, + sx, sy, side, side, + -bubbleSize / 2, -bubbleSize / 2, bubbleSize, bubbleSize, + ); + ctx.restore(); + + // Subtle ring around the bubble + ctx.beginPath(); + ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2); + ctx.strokeStyle = "rgba(255,255,255,0.25)"; + ctx.lineWidth = 2; + ctx.stroke(); + } + + animId = requestAnimationFrame(draw); + }; + draw(); + + return { + stream: canvas.captureStream(30), + stop: () => { + cancelAnimationFrame(animId); + screenVideo.pause(); + screenVideo.srcObject = null; + if (cameraVideo) { + cameraVideo.pause(); + cameraVideo.srcObject = null; + } + }, + }; +} + +// --------------------------------------------------------------------------- +// Hook +// --------------------------------------------------------------------------- + interface UseScreenRecorderOptions { + /** "video" = screen + webcam overlay + mic. "audio" = screen + mic only. */ + mode: RecordingMode; onFinish: (blob: Blob, durationMs: number, mimeType: string) => void; onError: (message: string) => void; } /** * Manages screen recording via Electron's desktopCapturer. - * Captures screen video + mic audio into a single MediaRecorder. - * Communicates with the floating control window via IPC. + * + * When mode is "video", captures screen video composited with a webcam + * overlay (via Canvas) plus mic audio. + * When mode is "audio", captures screen video with mic audio only. */ export function useScreenRecorder({ + mode, onFinish, onError, }: UseScreenRecorderOptions) { const recorderRef = useRef(null); const screenStreamRef = useRef(null); const micStreamRef = useRef(null); + const cameraStreamRef = useRef(null); + const compositorRef = useRef(null); const chunksRef = useRef([]); const startTimeRef = useRef(0); const cleanupIpcRef = useRef<(() => void) | null>(null); @@ -37,11 +150,18 @@ export function useScreenRecorder({ onErrorRef.current = onError; }); + const modeRef = useRef(mode); + modeRef.current = mode; + const stopAllTracks = useCallback(() => { + compositorRef.current?.stop(); + compositorRef.current = null; screenStreamRef.current?.getTracks().forEach((t) => t.stop()); micStreamRef.current?.getTracks().forEach((t) => t.stop()); + cameraStreamRef.current?.getTracks().forEach((t) => t.stop()); screenStreamRef.current = null; micStreamRef.current = null; + cameraStreamRef.current = null; recorderRef.current = null; chunksRef.current = []; cleanupIpcRef.current?.(); @@ -51,7 +171,9 @@ export function useScreenRecorder({ const startRecording = useCallback( async (sourceId: string) => { try { - // Screen video via Electron desktopCapturer + const includeCamera = modeRef.current === "video"; + + // 1. Screen video const screenStream = await navigator.mediaDevices.getUserMedia({ audio: false, video: { @@ -63,14 +185,31 @@ export function useScreenRecorder({ }); screenStreamRef.current = screenStream; - // Mic audio separately - const micStream = - await navigator.mediaDevices.getUserMedia({ audio: true }); + // 2. Mic audio + const micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); micStreamRef.current = micStream; - // Combine into a single stream + // 3. Camera (only in video mode) + let cameraStream: MediaStream | null = null; + if (includeCamera) { + try { + cameraStream = await navigator.mediaDevices.getUserMedia({ + video: { aspectRatio: { ideal: 1 }, width: { ideal: 320 } }, + audio: false, + }); + cameraStreamRef.current = cameraStream; + } catch { + // Camera unavailable — proceed without it + } + } + + // 4. Composite screen + camera via canvas + const compositor = createCompositor(screenStream, cameraStream); + compositorRef.current = compositor; + + // 5. Combine composited video + mic audio const combined = new MediaStream([ - ...screenStream.getVideoTracks(), + ...compositor.stream.getVideoTracks(), ...micStream.getAudioTracks(), ]); @@ -98,10 +237,10 @@ export function useScreenRecorder({ recorder.start(); - // Show the floating control window - window.electronScreen.startRecordingWindow(); + // 6. Show floating control window + window.electronScreen.startRecordingWindow({ includeCamera }); - // Listen for stop from floating window + // 7. Listen for stop from floating window cleanupIpcRef.current = window.electronScreen.onStopRequested(() => { if (recorderRef.current?.state === "recording") { recorderRef.current.stop(); diff --git a/js/src/features/particles/media-particle-view.tsx b/js/src/features/particles/media-particle-view.tsx index 990b6b1..78cdcaa 100644 --- a/js/src/features/particles/media-particle-view.tsx +++ b/js/src/features/particles/media-particle-view.tsx @@ -147,7 +147,7 @@ export const MediaParticleView = forwardRef {transcript && ( diff --git a/js/src/main.ts b/js/src/main.ts index 249824d..86fbd57 100644 --- a/js/src/main.ts +++ b/js/src/main.ts @@ -209,9 +209,30 @@ ipcMain.handle('screen:get-sources', async () => { }); // Screen recording IPC handlers -ipcMain.on('screen-record:start', () => { +ipcMain.on('screen-record:start', (_event, data: { includeCamera: boolean }) => { createScreenRecordWindow(); - screenRecordWindow?.showInactive(); + if (!screenRecordWindow) return; + + // Resize based on whether camera preview is shown + const winW = data.includeCamera ? 200 : 240; + const winH = data.includeCamera ? 176 : 48; + const { width, height } = screen.getPrimaryDisplay().workAreaSize; + screenRecordWindow.setSize(winW, winH); + screenRecordWindow.setPosition( + Math.round((width - winW) / 2), + height - winH - 32, + ); + + const send = () => { + screenRecordWindow?.webContents.send('screen-record:init', data); + screenRecordWindow?.showInactive(); + }; + + if (screenRecordWindow.webContents.isLoading()) { + screenRecordWindow.webContents.once('did-finish-load', send); + } else { + send(); + } }); ipcMain.on('screen-record:stop', () => { mainWindow?.webContents.send('screen-record:stopped'); diff --git a/js/src/preload.ts b/js/src/preload.ts index ef5f076..5f3579e 100644 --- a/js/src/preload.ts +++ b/js/src/preload.ts @@ -43,7 +43,7 @@ contextBridge.exposeInMainWorld('electronAutoplay', { contextBridge.exposeInMainWorld('electronScreen', { getScreenSources: () => ipcRenderer.invoke('screen:get-sources'), - startRecordingWindow: () => ipcRenderer.send('screen-record:start'), + startRecordingWindow: (data: { includeCamera: boolean }) => ipcRenderer.send('screen-record:start', data), stopRecordingWindow: () => ipcRenderer.send('screen-record:cancel'), onStopRequested: (callback: () => void) => { const handler = () => callback(); @@ -54,6 +54,11 @@ contextBridge.exposeInMainWorld('electronScreen', { contextBridge.exposeInMainWorld('electronScreenRecord', { stop: () => ipcRenderer.send('screen-record:stop'), + onInit: (callback: (data: { includeCamera: boolean }) => void) => { + const handler = (_event: Electron.IpcRendererEvent, data: { includeCamera: boolean }) => callback(data); + ipcRenderer.on('screen-record:init', handler); + return () => { ipcRenderer.removeListener('screen-record:init', handler); }; + }, }); contextBridge.exposeInMainWorld('electronLink', { diff --git a/js/src/screen_record_window/ScreenRecordControlApp.tsx b/js/src/screen_record_window/ScreenRecordControlApp.tsx index 41cac98..c3f6c16 100644 --- a/js/src/screen_record_window/ScreenRecordControlApp.tsx +++ b/js/src/screen_record_window/ScreenRecordControlApp.tsx @@ -1,9 +1,48 @@ -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { Square } from "lucide-react"; export function ScreenRecordControlApp() { const [elapsed, setElapsed] = useState(0); + const [includeCamera, setIncludeCamera] = useState(false); + const videoRef = useRef(null); + const streamRef = useRef(null); + // Listen for init data from main process + useEffect(() => { + return window.electronScreenRecord.onInit((data) => { + setIncludeCamera(data.includeCamera); + }); + }, []); + + // Acquire webcam for preview (independent from the recording capture) + useEffect(() => { + if (!includeCamera) return; + + let cancelled = false; + navigator.mediaDevices + .getUserMedia({ video: { aspectRatio: { ideal: 1 }, width: { ideal: 160 } }, audio: false }) + .then((stream) => { + if (cancelled) { + stream.getTracks().forEach((t) => t.stop()); + return; + } + streamRef.current = stream; + if (videoRef.current) { + videoRef.current.srcObject = stream; + } + }) + .catch(() => { + // Camera unavailable — just don't show preview + }); + + return () => { + cancelled = true; + streamRef.current?.getTracks().forEach((t) => t.stop()); + streamRef.current = null; + }; + }, [includeCamera]); + + // Timer useEffect(() => { const interval = setInterval(() => setElapsed((prev) => prev + 1), 1000); return () => clearInterval(interval); @@ -13,13 +52,33 @@ export function ScreenRecordControlApp() { const seconds = elapsed % 60; const display = `${minutes}:${seconds.toString().padStart(2, "0")}`; + const handleStop = useCallback(() => { + streamRef.current?.getTracks().forEach((t) => t.stop()); + streamRef.current = null; + window.electronScreenRecord.stop(); + }, []); + return ( -
-
+
+ {/* Webcam preview */} + {includeCamera && ( +
+
+ )} + + {/* Controls */} +
{display}
diff --git a/js/src/features/compose/compose-overlay.tsx b/js/src/features/compose/compose-overlay.tsx index c3e69c2..144e874 100644 --- a/js/src/features/compose/compose-overlay.tsx +++ b/js/src/features/compose/compose-overlay.tsx @@ -7,7 +7,7 @@ import { useScreenRecorder } from "@/features/compose/use-screen-recorder"; import { particlePath, parseParticlePath } from "@/lib/particle-path"; import type { ParticlePath } from "@/lib/particle-path"; import { RecordingOverlay } from "@/features/compose/recording-overlay"; -import { ScreenSourcePicker } from "@/features/compose/screen-source-picker"; +import { ScreenSourcePicker } from "@/components/screen-source-picker"; import { TextComposeStep } from "@/features/compose/text-compose-step"; import { ConfigureStreamStep } from "@/features/compose/configure-stream-step"; import { apiClient } from "@/api/client"; @@ -485,6 +485,9 @@ export function ComposeOverlay({ <> {step === "picking" && ( diff --git a/js/src/huddle_window/HuddleApp.tsx b/js/src/huddle_window/HuddleApp.tsx index 499d8b4..94d8b24 100644 --- a/js/src/huddle_window/HuddleApp.tsx +++ b/js/src/huddle_window/HuddleApp.tsx @@ -22,7 +22,7 @@ import { } from '@livekit/components-react'; import { RoomEvent, Track } from 'livekit-client'; import { useState, useEffect, useRef, useCallback } from 'react'; -import { ScreenPicker } from './ScreenPicker'; +import { ScreenSourcePicker } from '@/components/screen-source-picker'; export function HuddleApp() { const [connection, setConnection] = useState<{ token: string; serverUrl: string } | null>(null); @@ -198,7 +198,10 @@ function HuddleContent() { {showPicker && ( - setShowPicker(false)} /> diff --git a/js/src/huddle_window/ScreenPicker.tsx b/js/src/huddle_window/ScreenPicker.tsx deleted file mode 100644 index 4cfea99..0000000 --- a/js/src/huddle_window/ScreenPicker.tsx +++ /dev/null @@ -1,117 +0,0 @@ -import { useState, useEffect } from 'react'; - -interface ScreenPickerProps { - onSelect: (sourceId: string) => void; - onCancel: () => void; -} - -export function ScreenPicker({ onSelect, onCancel }: ScreenPickerProps) { - const [sources, setSources] = useState([]); - const [selectedId, setSelectedId] = useState(null); - const [loading, setLoading] = useState(true); - - useEffect(() => { - window.electronHuddle.getScreenSources().then((result) => { - setSources(result); - setLoading(false); - }); - }, []); - - const screens = sources.filter((s) => s.id.startsWith('screen:')); - const windows = sources.filter((s) => s.id.startsWith('window:')); - - return ( -
-
-
-

Share your screen

- -
- -
- {loading ? ( -

Loading sources…

- ) : ( - <> - {screens.length > 0 && ( - - )} - {windows.length > 0 && ( - - )} - - )} -
- -
- - -
-
-
- ); -} - -function SourceSection({ - title, - sources, - selectedId, - onSelect, -}: { - title: string; - sources: ScreenSource[]; - selectedId: string | null; - onSelect: (id: string) => void; -}) { - return ( -
-

{title}

-
- {sources.map((source) => ( - - ))} -
-
- ); -} diff --git a/js/src/styles/globals.css b/js/src/styles/globals.css index 928dc89..2e23250 100644 --- a/js/src/styles/globals.css +++ b/js/src/styles/globals.css @@ -2,6 +2,12 @@ @import "tw-animate-css"; @import "shadcn/tailwind.css"; +/* Ensure Tailwind scans shared components/features used by secondary windows + (huddle, screen_record, autoplay) whose vite root is a subdirectory. */ +@source "../components"; +@source "../features"; +@source "../lib"; + @custom-variant dark (&:is(.dark *)); @theme inline { -- 2.54.0 From acaedff73cd1352e8104f427684fbbba2a176670 Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 8 Apr 2026 17:13:47 -0700 Subject: [PATCH 10/10] fix: improve visibility of keyboard hints During compose video or screen recording, the keyboard hints were invisible if the content was super bright. --- js/src/features/compose/recording-overlay.tsx | 5 +++++ js/src/features/particles/stream-view.tsx | 3 +++ 2 files changed, 8 insertions(+) diff --git a/js/src/features/compose/recording-overlay.tsx b/js/src/features/compose/recording-overlay.tsx index bf69b41..195d2cd 100644 --- a/js/src/features/compose/recording-overlay.tsx +++ b/js/src/features/compose/recording-overlay.tsx @@ -191,6 +191,11 @@ export function RecordingOverlay({ /> )} + {/* Bottom gradient scrim for keyboard hint readability */} + {(isRecording || isReviewing) && !isLoading && ( +
+ )} + {/* Top center: recording indicator */}
{isRecording && !isLoading ? ( diff --git a/js/src/features/particles/stream-view.tsx b/js/src/features/particles/stream-view.tsx index e2db8a1..68a8d2b 100644 --- a/js/src/features/particles/stream-view.tsx +++ b/js/src/features/particles/stream-view.tsx @@ -436,6 +436,9 @@ export function StreamView({ path, streamParticle }: StreamViewProps) { disabled={streamParticle.status === "closed"} /> + {/* Bottom gradient safe zone for keyboard hints */} +
+ {/* BottomBar */}