feat: send screen recordings (#128)

* first pass implementation * remove screenrecord shortcut hint * refactor * fix: prevent mirror review of screen recording * feat: allow recording with webcam overlay * fix: review screen recording without clipped content * tidy keyboard hints consistent position * refactor: re-use one component for screen picker huddles and screen clips use same picker component now. We had to make sure that tailwind works for both of them. * fix: improve visibility of keyboard hints During compose video or screen recording, the keyboard hints were invisible if the content was super bright.
2026-04-08 17:14:39 -07:00
parent bebd814c46
commit be7a42c483
18 changed files with 694 additions and 29 deletions
@@ -3,9 +3,11 @@ import { toast } from "sonner";
 import { useAuthStore } from "@/stores/auth-store";
 import { useCreateParticle, useCreateStreamParticle } from "@/hooks/use-create-particle";
 import { useRecorder } from "@/features/compose/use-recorder";
+import { useScreenRecorder } from "@/features/compose/use-screen-recorder";
 import { particlePath, parseParticlePath } from "@/lib/particle-path";
 import type { ParticlePath } from "@/lib/particle-path";
 import { RecordingOverlay } from "@/features/compose/recording-overlay";
+import { ScreenSourcePicker } from "@/components/screen-source-picker";
 import { TextComposeStep } from "@/features/compose/text-compose-step";
 import { ConfigureStreamStep } from "@/features/compose/configure-stream-step";
 import { apiClient } from "@/api/client";
@@ -15,7 +17,9 @@ import { createImageThumbnail } from "@/lib/image-thumbnail";
 import { MAX_ATTACHMENT_SIZE_BYTES, MAX_ATTACHMENTS } from "@/lib/constants";
 import type { PendingAttachment } from "@/features/compose/attachment-strip";

-type ComposeStep = "idle" | "recording" | "reviewing" | "typing" | "configuring" | "submitting";
+type ComposeStep = "idle" | "picking" | "recording" | "reviewing" | "typing" | "configuring" | "submitting";
+
+type RecordingSource = "media" | "screen";

 interface ComposeOverlayProps {
  networkId: string;
@@ -50,6 +54,7 @@ export function ComposeOverlay({
  const [reviewMimeType, setReviewMimeType] = useState<string | null>(null);

  const [attachments, setAttachments] = useState<PendingAttachment[]>([]);
+  const [recordingSource, setRecordingSource] = useState<RecordingSource>("media");

  const recordingMode = useMediaSettingsStore((s) => s.recordingMode);
  const userId = useAuthStore((s) => s.user?.id);
@@ -61,6 +66,8 @@ export function ComposeOverlay({
  const recordStartRef = useRef(0);
  const disabledRef = useRef(disabled);
  disabledRef.current = disabled;
+  const recordingSourceRef = useRef(recordingSource);
+  recordingSourceRef.current = recordingSource;

  const setStepSync = useCallback((next: ComposeStep) => {
    stepRef.current = next;
@@ -86,6 +93,7 @@ export function ComposeOverlay({
    setReviewBlob(null);
    setReviewDurationMs(0);
    setReviewMimeType(null);
+    setRecordingSource("media");
    setAttachments((prev) => {
      revokeAttachmentThumbnails(prev);
      return [];
@@ -151,6 +159,24 @@ export function ComposeOverlay({
    onError: (message) => setError(message),
  });

+  const {
+    startRecording: startScreenRecording,
+    stopRecording: stopScreenRecording,
+    cancelRecording: cancelScreenRecording,
+  } = useScreenRecorder({
+    mode: recordingMode,
+    onFinish: (blob, durationMs, mimeType) => {
+      setStepSync("reviewing");
+      setReviewBlob(blob);
+      setReviewDurationMs(durationMs);
+      setReviewMimeType(mimeType);
+    },
+    onError: (message) => {
+      setError(message);
+      cancel();
+    },
+  });
+
  // --- Submission ---

  const uploadMedia = useCallback(
@@ -259,6 +285,7 @@ export function ComposeOverlay({
          reviewMimeType,
        );

+        const isAudioOnly = reviewMimeType.startsWith("audio/");
        particleId = await createParticle.mutateAsync({
          path,
          type: "media",
@@ -267,6 +294,9 @@ export function ComposeOverlay({
            mime_type: reviewMimeType,
            duration_ms: reviewDurationMs,
            size_bytes,
+            ...(!isAudioOnly && {
+              source: recordingSource === "screen" ? "screen" as const : "camera" as const,
+            }),
          },
          createdByHumanId: userId,
        });
@@ -283,6 +313,7 @@ export function ComposeOverlay({
      reviewBlob,
      reviewMimeType,
      reviewDurationMs,
+      recordingSource,
      createParticle,
      uploadMedia,
      uploadAttachments,
@@ -327,7 +358,7 @@ export function ComposeOverlay({
    const handleKeyDown = (e: KeyboardEvent) => {
      const currentStep = stepRef.current;

-      if (currentStep === "typing" || currentStep === "configuring") {
+      if (currentStep === "typing" || currentStep === "configuring" || currentStep === "picking") {
        if (e.key === "Escape") {
          e.preventDefault();
          cancel();
@@ -347,7 +378,7 @@ export function ComposeOverlay({
      switch (currentStep) {
        case "idle": {
          if (disabledRef.current) {
-            if ((e.key === "`" && !e.repeat) || e.key === "t" || e.key === "T") {
+            if ((e.key === "`" && !e.repeat) || e.key === "t" || e.key === "T" || e.key === "s" || e.key === "S") {
              e.preventDefault();
              toast.info("This stream is closed");
            }
@@ -356,8 +387,13 @@ export function ComposeOverlay({
          if (e.key === "`" && !e.repeat) {
            e.preventDefault();
            recordStartRef.current = Date.now();
+            setRecordingSource("media");
            setStepSync("recording");
            startRecording();
+          } else if (e.key === "s" || e.key === "S") {
+            e.preventDefault();
+            setRecordingSource("screen");
+            setStepSync("picking");
          } else if (e.key === "t" || e.key === "T") {
            e.preventDefault();
            setStepSync("typing");
@@ -370,9 +406,17 @@ export function ComposeOverlay({
            // Second tap stops recording (toggle mode)
            e.preventDefault();
            stopRecording();
+          } else if ((e.key === "s" || e.key === "S") && recordingSourceRef.current === "screen") {
+            // S stops screen recording when main window is focused
+            e.preventDefault();
+            stopScreenRecording();
          } else if (e.key === "q" || e.key === "Q" || e.key === "Escape") {
            e.preventDefault();
-            cancelRecording();
+            if (recordingSourceRef.current === "screen") {
+              cancelScreenRecording();
+            } else {
+              cancelRecording();
+            }
            cancel();
          }
          break;
@@ -381,7 +425,11 @@ export function ComposeOverlay({
        case "reviewing": {
          if (e.key === "q" || e.key === "Q" || e.key === "Escape") {
            e.preventDefault();
-            cancelRecording();
+            if (recordingSourceRef.current === "screen") {
+              cancelScreenRecording();
+            } else {
+              cancelRecording();
+            }
            cancel();
          } else if (e.key === "Enter") {
            e.preventDefault();
@@ -397,7 +445,7 @@ export function ComposeOverlay({
    };

    const handleKeyUp = (e: KeyboardEvent) => {
-      if (stepRef.current === "recording" && e.key === "`") {
+      if (stepRef.current === "recording" && e.key === "`" && recordingSourceRef.current === "media") {
        e.preventDefault();
        // Only stop on release if held long enough (hold-to-record mode).
        // Quick taps are handled by the second keydown (toggle mode).
@@ -413,7 +461,17 @@ export function ComposeOverlay({
      window.removeEventListener("keydown", handleKeyDown);
      window.removeEventListener("keyup", handleKeyUp);
    };
-  }, [targetPath, startRecording, stopRecording, cancelRecording, cancel, setStepSync]);
+  }, [targetPath, startRecording, stopRecording, cancelRecording, startScreenRecording, stopScreenRecording, cancelScreenRecording, cancel, setStepSync]);
+
+  // --- Screen source selection handler ---
+
+  const handleScreenSourceSelected = useCallback(
+    (sourceId: string) => {
+      setStepSync("recording");
+      startScreenRecording(sourceId);
+    },
+    [setStepSync, startScreenRecording],
+  );

  // --- Render ---

@@ -425,7 +483,16 @@ export function ComposeOverlay({

  return (
    <>
-      {(step === "recording" || step === "reviewing") && (
+      {step === "picking" && (
+        <ScreenSourcePicker
+          title="Record your screen"
+          confirmLabel="Record"
+          getSources={window.electronScreen.getScreenSources}
+          onSelect={handleScreenSourceSelected}
+          onCancel={cancel}
+        />
+      )}
+      {(step === "recording" || step === "reviewing") && recordingSource === "media" && (
        <RecordingOverlay
          step={step}
          mediaStream={mediaStream}
@@ -438,6 +505,25 @@ export function ComposeOverlay({
          onAddFiles={openFilePicker}
          isDragging={isDragging}
          dropZoneProps={dropZoneProps}
+          mirror={true}
+          objectFit="cover"
+        />
+      )}
+      {step === "reviewing" && recordingSource === "screen" && reviewBlob && (
+        <RecordingOverlay
+          step="reviewing"
+          mediaStream={null}
+          recordingMode="video"
+          reviewBlob={reviewBlob}
+          error={error}
+          onClose={cancel}
+          attachments={attachments}
+          onRemoveAttachment={removeAttachment}
+          onAddFiles={openFilePicker}
+          isDragging={isDragging}
+          dropZoneProps={dropZoneProps}
+          mirror={false}
+          objectFit="contain"
        />
      )}
      {step === "typing" && (
@@ -25,6 +25,10 @@ interface RecordingOverlayProps {
    onDragLeave: (e: React.DragEvent) => void;
    onDrop: (e: React.DragEvent) => void;
  };
+  /** Mirror the video horizontally. Defaults to true (selfie-view for webcam). */
+  mirror?: boolean;
+  /** How video fills its container. Defaults to "cover". Use "contain" for screen recordings. */
+  objectFit?: "cover" | "contain";
 }

 function RecordingTimer() {
@@ -52,9 +56,13 @@ function RecordingTimer() {
 function ReviewPlayback({
  blob,
  isVideo,
+  mirror = true,
+  objectFit = "cover",
 }: {
  blob: Blob;
  isVideo: boolean;
+  mirror?: boolean;
+  objectFit?: "cover" | "contain";
 }) {
  const urlRef = useRef<string | null>(null);
  const [objectUrl, setObjectUrl] = useState<string | null>(null);
@@ -82,7 +90,7 @@ function ReviewPlayback({
        autoPlay
        loop
        playsInline
-        className="absolute inset-0 h-full w-full -scale-x-100 object-cover"
+        className={`absolute inset-0 h-full w-full ${objectFit === "contain" ? "object-contain" : "object-cover"}${mirror ? " -scale-x-100" : ""}`}
      />
    );
  }
@@ -119,6 +127,8 @@ export function RecordingOverlay({
  onAddFiles,
  isDragging,
  dropZoneProps,
+  mirror = true,
+  objectFit = "cover",
 }: RecordingOverlayProps) {
  const videoRef = useRef<HTMLVideoElement>(null);
  const recordingAudioSource = useAudioSource(mediaStream ?? null);
@@ -176,9 +186,16 @@ export function RecordingOverlay({
        <ReviewPlayback
          blob={reviewBlob}
          isVideo={recordingMode === "video"}
+          mirror={mirror}
+          objectFit={objectFit}
        />
      )}

+      {/* Bottom gradient scrim for keyboard hint readability */}
+      {(isRecording || isReviewing) && !isLoading && (
+        <div className="pointer-events-none absolute inset-x-0 bottom-0 z-[5] h-1/4 bg-gradient-to-t from-black/60 to-transparent" />
+      )}
+
      {/* Top center: recording indicator */}
      <div className="absolute top-8 z-10">
        {isRecording && !isLoading ? (
@@ -199,7 +216,7 @@ export function RecordingOverlay({

      {/* Bottom center: keyboard hints */}
      {isRecording && !isLoading && (
-        <div className="absolute bottom-8 z-10 flex items-center gap-4 text-sm text-white/50">
+        <div className="absolute bottom-4 z-10 flex items-center gap-4 text-sm text-white/50">
          <span>
            Release{" "}
            <kbd className="rounded bg-white/10 px-1.5 py-0.5 font-mono text-xs">
@@ -217,7 +234,7 @@ export function RecordingOverlay({
      )}

      {isReviewing && (
-        <div className="absolute bottom-8 z-10 flex flex-col items-center gap-3">
+        <div className="absolute bottom-4 z-10 flex flex-col items-center gap-3">
          {attachments.length > 0 && (
            <div className="px-4">
              <AttachmentStrip
@@ -0,0 +1,287 @@
+import { useCallback, useEffect, useRef } from "react";
+import type { RecordingMode } from "@/stores/media-settings-store";
+
+const VIDEO_PREFERRED_MIME = "video/webm;codecs=vp9,opus";
+const VIDEO_FALLBACK_MIME = "video/webm";
+
+function getScreenMime(): string {
+  return MediaRecorder.isTypeSupported(VIDEO_PREFERRED_MIME)
+    ? VIDEO_PREFERRED_MIME
+    : VIDEO_FALLBACK_MIME;
+}
+
+// ---------------------------------------------------------------------------
+// Canvas compositor — overlays webcam as a circular PiP on the screen feed
+// ---------------------------------------------------------------------------
+
+interface Compositor {
+  /** Composited video stream (screen + optional webcam bubble). */
+  stream: MediaStream;
+  /** Tear down the animation loop and video elements. */
+  stop: () => void;
+}
+
+function createCompositor(
+  screenStream: MediaStream,
+  cameraStream: MediaStream | null,
+): Compositor {
+  const screenTrack = screenStream.getVideoTracks()[0];
+  const settings = screenTrack.getSettings();
+  const width = settings.width ?? 1920;
+  const height = settings.height ?? 1080;
+
+  const canvas = document.createElement("canvas");
+  canvas.width = width;
+  canvas.height = height;
+  const ctx = canvas.getContext("2d")!;
+
+  // Hidden video elements used as frame sources
+  const screenVideo = document.createElement("video");
+  screenVideo.srcObject = screenStream;
+  screenVideo.muted = true;
+  screenVideo.playsInline = true;
+  screenVideo.play();
+
+  let cameraVideo: HTMLVideoElement | null = null;
+  if (cameraStream) {
+    cameraVideo = document.createElement("video");
+    cameraVideo.srcObject = cameraStream;
+    cameraVideo.muted = true;
+    cameraVideo.playsInline = true;
+    cameraVideo.play();
+  }
+
+  let animId = 0;
+
+  const draw = () => {
+    // Screen — full canvas
+    ctx.drawImage(screenVideo, 0, 0, width, height);
+
+    // Webcam — circular bubble in bottom-left
+    if (cameraVideo && cameraVideo.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
+      const bubbleSize = Math.round(Math.min(width, height) * 0.18);
+      const margin = Math.round(bubbleSize * 0.3);
+      const cx = margin + bubbleSize / 2;
+      const cy = height - margin - bubbleSize / 2;
+
+      ctx.save();
+      ctx.beginPath();
+      ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2);
+      ctx.clip();
+
+      // Crop camera to square centre, mirror horizontally
+      const vw = cameraVideo.videoWidth || 1;
+      const vh = cameraVideo.videoHeight || 1;
+      const side = Math.min(vw, vh);
+      const sx = (vw - side) / 2;
+      const sy = (vh - side) / 2;
+
+      ctx.translate(cx, cy);
+      ctx.scale(-1, 1); // horizontal flip
+      ctx.drawImage(
+        cameraVideo,
+        sx, sy, side, side,
+        -bubbleSize / 2, -bubbleSize / 2, bubbleSize, bubbleSize,
+      );
+      ctx.restore();
+
+      // Subtle ring around the bubble
+      ctx.beginPath();
+      ctx.arc(cx, cy, bubbleSize / 2, 0, Math.PI * 2);
+      ctx.strokeStyle = "rgba(255,255,255,0.25)";
+      ctx.lineWidth = 2;
+      ctx.stroke();
+    }
+
+    animId = requestAnimationFrame(draw);
+  };
+  draw();
+
+  return {
+    stream: canvas.captureStream(30),
+    stop: () => {
+      cancelAnimationFrame(animId);
+      screenVideo.pause();
+      screenVideo.srcObject = null;
+      if (cameraVideo) {
+        cameraVideo.pause();
+        cameraVideo.srcObject = null;
+      }
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Hook
+// ---------------------------------------------------------------------------
+
+interface UseScreenRecorderOptions {
+  /** "video" = screen + webcam overlay + mic. "audio" = screen + mic only. */
+  mode: RecordingMode;
+  onFinish: (blob: Blob, durationMs: number, mimeType: string) => void;
+  onError: (message: string) => void;
+}
+
+/**
+ * Manages screen recording via Electron's desktopCapturer.
+ *
+ * When mode is "video", captures screen video composited with a webcam
+ * overlay (via Canvas) plus mic audio.
+ * When mode is "audio", captures screen video with mic audio only.
+ */
+export function useScreenRecorder({
+  mode,
+  onFinish,
+  onError,
+}: UseScreenRecorderOptions) {
+  const recorderRef = useRef<MediaRecorder | null>(null);
+  const screenStreamRef = useRef<MediaStream | null>(null);
+  const micStreamRef = useRef<MediaStream | null>(null);
+  const cameraStreamRef = useRef<MediaStream | null>(null);
+  const compositorRef = useRef<Compositor | null>(null);
+  const chunksRef = useRef<Blob[]>([]);
+  const startTimeRef = useRef<number>(0);
+  const cleanupIpcRef = useRef<(() => void) | null>(null);
+
+  const onFinishRef = useRef(onFinish);
+  const onErrorRef = useRef(onError);
+  useEffect(() => {
+    onFinishRef.current = onFinish;
+    onErrorRef.current = onError;
+  });
+
+  const modeRef = useRef(mode);
+  modeRef.current = mode;
+
+  const stopAllTracks = useCallback(() => {
+    compositorRef.current?.stop();
+    compositorRef.current = null;
+    screenStreamRef.current?.getTracks().forEach((t) => t.stop());
+    micStreamRef.current?.getTracks().forEach((t) => t.stop());
+    cameraStreamRef.current?.getTracks().forEach((t) => t.stop());
+    screenStreamRef.current = null;
+    micStreamRef.current = null;
+    cameraStreamRef.current = null;
+    recorderRef.current = null;
+    chunksRef.current = [];
+    cleanupIpcRef.current?.();
+    cleanupIpcRef.current = null;
+  }, []);
+
+  const startRecording = useCallback(
+    async (sourceId: string) => {
+      try {
+        const includeCamera = modeRef.current === "video";
+
+        // 1. Screen video
+        const screenStream = await navigator.mediaDevices.getUserMedia({
+          audio: false,
+          video: {
+            mandatory: {
+              chromeMediaSource: "desktop",
+              chromeMediaSourceId: sourceId,
+            },
+          } as unknown as MediaTrackConstraints,
+        });
+        screenStreamRef.current = screenStream;
+
+        // 2. Mic audio
+        const micStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+        micStreamRef.current = micStream;
+
+        // 3. Camera (only in video mode)
+        let cameraStream: MediaStream | null = null;
+        if (includeCamera) {
+          try {
+            cameraStream = await navigator.mediaDevices.getUserMedia({
+              video: { aspectRatio: { ideal: 1 }, width: { ideal: 320 } },
+              audio: false,
+            });
+            cameraStreamRef.current = cameraStream;
+          } catch {
+            // Camera unavailable — proceed without it
+          }
+        }
+
+        // 4. Composite screen + camera via canvas
+        const compositor = createCompositor(screenStream, cameraStream);
+        compositorRef.current = compositor;
+
+        // 5. Combine composited video + mic audio
+        const combined = new MediaStream([
+          ...compositor.stream.getVideoTracks(),
+          ...micStream.getAudioTracks(),
+        ]);
+
+        chunksRef.current = [];
+        startTimeRef.current = Date.now();
+
+        const mime = getScreenMime();
+        const recorder = new MediaRecorder(combined, { mimeType: mime });
+        recorderRef.current = recorder;
+
+        recorder.ondataavailable = (e) => {
+          if (e.data.size > 0) chunksRef.current.push(e.data);
+        };
+
+        recorder.onstop = () => {
+          const durationMs = Date.now() - startTimeRef.current;
+          const blob = new Blob(chunksRef.current, { type: mime });
+          stopAllTracks();
+          window.electronScreen.stopRecordingWindow();
+
+          if (blob.size > 0) {
+            onFinishRef.current(blob, durationMs, mime);
+          }
+        };
+
+        recorder.start();
+
+        // 6. Show floating control window
+        window.electronScreen.startRecordingWindow({ includeCamera });
+
+        // 7. Listen for stop from floating window
+        cleanupIpcRef.current = window.electronScreen.onStopRequested(() => {
+          if (recorderRef.current?.state === "recording") {
+            recorderRef.current.stop();
+          }
+        });
+      } catch (err) {
+        stopAllTracks();
+        window.electronScreen.stopRecordingWindow();
+        onErrorRef.current(
+          err instanceof Error ? err.message : "Failed to start screen recording",
+        );
+      }
+    },
+    [stopAllTracks],
+  );
+
+  const stopRecording = useCallback(() => {
+    if (recorderRef.current?.state === "recording") {
+      recorderRef.current.stop();
+    }
+  }, []);
+
+  const cancelRecording = useCallback(() => {
+    if (recorderRef.current) {
+      recorderRef.current.ondataavailable = null;
+      recorderRef.current.onstop = null;
+      if (recorderRef.current.state === "recording") {
+        recorderRef.current.stop();
+      }
+    }
+    stopAllTracks();
+    window.electronScreen.stopRecordingWindow();
+  }, [stopAllTracks]);
+
+  // Cleanup on unmount
+  useEffect(() => {
+    return () => {
+      stopAllTracks();
+      window.electronScreen.stopRecordingWindow();
+    };
+  }, [stopAllTracks]);
+
+  return { startRecording, stopRecording, cancelRecording };
+}
@@ -147,7 +147,7 @@ export const MediaParticleView = forwardRef<MediaParticleHandle, MediaParticleVi
        playsInline
        onEnded={onEnded}
        onTimeUpdate={handleTimeUpdate}
-        className="h-full w-full object-cover"
+        className={`h-full w-full ${particle.properties.source === "screen" ? "object-contain bg-black" : "object-cover"}`}
      />

      {transcript && (
@@ -130,6 +130,7 @@ const STREAM_VIEW_KEYBINDINGS: KeybindingGroup[] = [
    label: "Compose",
    bindings: [
      { keys: ["Hold", "`"], description: "Reply" },
+      { keys: ["S"], description: "Screen record" },
      { keys: ["T"], description: "Text compose" },
      { keys: ["H"], description: "Join huddle" },
    ],
@@ -435,6 +436,9 @@ export function StreamView({ path, streamParticle }: StreamViewProps) {
        disabled={streamParticle.status === "closed"}
      />

+      {/* Bottom gradient safe zone for keyboard hints */}
+      <div className="pointer-events-none absolute inset-x-0 bottom-0 z-[5] h-1/4 bg-gradient-to-t from-black/60 to-transparent" />
+
      {/* BottomBar */}
      <BottomBar
        visible={controlsVisible}