feat: captions on mobile

Closes #199

js/mobile/src/features/stream-view/MediaParticleView.tsx

@@ -7,6 +7,9 @@ import type { Particle } from "@/api/types";
 import { apiClient } from "@/api/client";
 import { logError } from "@/lib/errors";
 import { useEvent } from "@/hooks/use-event";
+import { useTranscriptPlayback } from "@/hooks/use-transcript-playback";
+import { TranscriptOverlay } from "./TranscriptOverlay";
+import { useStreamSafeArea } from "./stream-safe-area";
 
 type MediaParticle = Extract<Particle, { type: "media" }>;
 
@@ -90,6 +93,7 @@ function PlayableMediaView({
 }) {
   const [sourceUri, setSourceUri] = useState<string | null>(null);
   const [resolveError, setResolveError] = useState<Error | null>(null);
+  const [currentTime, setCurrentTime] = useState(0);
 
   // Fetch the signed download URL once per active object. Orion URLs are
   // time-limited — we treat the URL as one-shot for this view's lifetime.
@@ -99,6 +103,7 @@ function PlayableMediaView({
     let cancelled = false;
     setSourceUri(null);
     setResolveError(null);
+    setCurrentTime(0);
     apiClient
       .getParticleDownloadUrl(activeObjectId)
       .then((url) => {
@@ -143,6 +148,24 @@ function PlayableMediaView({
     }
   });
 
+  // Drive caption highlighting from the player's own timeUpdate cadence
+  // (timeUpdateEventInterval = 0.15s above). Pausing halts the events, which
+  // naturally freezes the active word/sentence — no extra plumbing needed.
+  useEventListener(player, "timeUpdate", ({ currentTime: t }) => {
+    setCurrentTime(t);
+  });
+
+  const transcript = particle.properties.transcript;
+  const { activeSentence, activeWordIndex } = useTranscriptPlayback(
+    transcript,
+    currentTime,
+  );
+  const safeArea = useStreamSafeArea();
+  // Sit just above the compose dock. Pills moved to the top of the screen so
+  // the only thing this offset has to clear is the dock itself plus a small
+  // gap — captions can ride lower than they did before.
+  const captionBottomOffset = safeArea.bottom;
+
   const onEndedStable = useEvent(onEnded);
   const onProgressStable = useEvent(onProgress);
 
@@ -208,6 +231,14 @@ function PlayableMediaView({
         <Text className="text-white mt-6 text-lg font-medium">
           Voice message
         </Text>
+        {transcript ? (
+          <TranscriptOverlay
+            transcript={transcript}
+            activeSentence={activeSentence}
+            activeWordIndex={activeWordIndex}
+            bottomOffset={captionBottomOffset}
+          />
+        ) : null}
       </View>
     );
   }
@@ -225,6 +256,14 @@ function PlayableMediaView({
         allowsFullscreen={false}
         allowsPictureInPicture={false}
       />
+      {transcript ? (
+        <TranscriptOverlay
+          transcript={transcript}
+          activeSentence={activeSentence}
+          activeWordIndex={activeWordIndex}
+          bottomOffset={captionBottomOffset}
+        />
+      ) : null}
     </View>
   );
 }

js/mobile/src/features/stream-view/StreamView.tsx

@@ -370,10 +370,11 @@ function StreamViewInner({ streamParticle, path, onExit }: StreamViewProps) {
   const exitRemainingMs = useExitCountdown(status, paused, exit);
 
   // Chrome reservations: top = safe-area + segmented bar (3) + gap (12) +
-  // metadata row (~38) + breathing room (12). Bottom = safe-area + room for
-  // pause / countdown pills + the compose dock that lands in this same step.
+  // metadata row (~38) + breathing room (12). Bottom = safe-area + compose
+  // dock + breathing room. Pause / countdown pills moved to the top so the
+  // bottom only reserves space for the compose dock now.
   const chromeTop = insets.top + 65;
-  const chromeBottom = insets.bottom + 96;
+  const chromeBottom = insets.bottom + COMPOSE_DOCK_HEIGHT + 14;
 
   // --- Render the active particle ---
   const renderParticle = (particle: Particle) => {
@@ -499,12 +500,14 @@ function StreamViewInner({ streamParticle, path, onExit }: StreamViewProps) {
               </View>
             </View>
 
-            {/* Bottom chrome: paused pill + exit countdown. Sit above the
-                compose dock so the record button doesn't cover them. */}
+            {/* Top status pills: paused + exit countdown. Anchored just below
+                the metadata row (avatar + name ≈ 40px tall, starts at
+                insets.top + 32) so they share the top chrome real estate
+                instead of competing with captions at the bottom. */}
             <View
               pointerEvents="none"
-              className="absolute inset-x-0 bottom-0 items-center"
-              style={{ paddingBottom: insets.bottom + COMPOSE_DOCK_HEIGHT }}
+              className="absolute inset-x-0 items-center"
+              style={{ top: insets.top + 88 }}
             >
               {paused ? (
                 <View className="bg-white/15 rounded-full px-3 py-1">

js/mobile/src/features/stream-view/TranscriptOverlay.tsx

@@ -0,0 +1,102 @@
+import { useMemo, useRef } from "react";
+import { Text, View } from "react-native";
+import type { Transcript } from "@/api/types";
+
+type Sentence = Transcript["paragraphs"][number]["sentences"][number];
+type Word = Transcript["words"][number];
+
+const CHUNK_SIZE = 9;
+
+function chunkWords(words: Word[]): Word[][] {
+  const chunks: Word[][] = [];
+  for (let i = 0; i < words.length; i += CHUNK_SIZE) {
+    chunks.push(words.slice(i, i + CHUNK_SIZE));
+  }
+  return chunks;
+}
+
+interface TranscriptOverlayProps {
+  transcript: Transcript;
+  activeSentence: Sentence | null;
+  activeWordIndex: number | null;
+  /** Distance from the bottom of the parent container, in px. */
+  bottomOffset: number;
+}
+
+export function TranscriptOverlay({
+  transcript,
+  activeSentence,
+  activeWordIndex,
+  bottomOffset,
+}: TranscriptOverlayProps) {
+  const sentenceWords = useMemo(() => {
+    if (!activeSentence) return [];
+    return transcript.words.filter(
+      (w) => w.start >= activeSentence.start && w.end <= activeSentence.end,
+    );
+  }, [transcript.words, activeSentence]);
+
+  const chunks = useMemo(() => chunkWords(sentenceWords), [sentenceWords]);
+
+  const activeWord =
+    activeWordIndex !== null ? transcript.words[activeWordIndex] : null;
+
+  const lastSpokenWordRef = useRef<Word | null>(null);
+  if (activeWord) {
+    lastSpokenWordRef.current = activeWord;
+  }
+  const highlightWord = activeWord ?? lastSpokenWordRef.current;
+
+  const lastChunkRef = useRef<Word[] | null>(null);
+
+  const activeChunk = useMemo(() => {
+    if (activeWord) {
+      for (const chunk of chunks) {
+        if (
+          chunk.some(
+            (w) => w.start === activeWord.start && w.end === activeWord.end,
+          )
+        ) {
+          lastChunkRef.current = chunk;
+          return chunk;
+        }
+      }
+    }
+    if (lastChunkRef.current && chunks.some((c) => c === lastChunkRef.current)) {
+      return lastChunkRef.current;
+    }
+    const fallback = chunks[0] ?? null;
+    lastChunkRef.current = fallback;
+    return fallback;
+  }, [chunks, activeWord]);
+
+  if (!activeSentence || !activeChunk || activeChunk.length === 0) return null;
+
+  return (
+    <View
+      pointerEvents="none"
+      className="absolute left-0 right-0 items-center px-6"
+      style={{ bottom: bottomOffset }}
+    >
+      <View className="rounded-lg px-5 py-3 max-w-[480px]">
+        <Text className="text-[24px] leading-[44px] text-center">
+          {activeChunk.map((word, i) => {
+            const isSpoken =
+              highlightWord !== null && word.start <= highlightWord.end;
+            return (
+              <Text
+                key={`${word.start}-${i}`}
+                className={
+                  isSpoken ? "text-white font-medium" : "text-white/40"
+                }
+              >
+                {i > 0 ? " " : ""}
+                {word.word}
+              </Text>
+            );
+          })}
+        </Text>
+      </View>
+    </View>
+  );
+}

js/mobile/src/hooks/use-transcript-playback.ts

@@ -0,0 +1,49 @@
+import { useMemo } from "react";
+import type { Transcript } from "@/api/types";
+
+type Sentence = Transcript["paragraphs"][number]["sentences"][number];
+
+interface TranscriptPlaybackState {
+  /** The sentence currently being spoken, or null if between sentences */
+  activeSentence: Sentence | null;
+  /** Index of the active word within the transcript's flat words array */
+  activeWordIndex: number | null;
+}
+
+export function useTranscriptPlayback(
+  transcript: Transcript | undefined,
+  currentTime: number,
+): TranscriptPlaybackState {
+  return useMemo(() => {
+    if (!transcript) return { activeSentence: null, activeWordIndex: null };
+
+    // Find the active sentence across all paragraphs
+    let activeSentence: Sentence | null = null;
+    for (const paragraph of transcript.paragraphs) {
+      const sentence = paragraph.sentences.find(
+        (s) => currentTime >= s.start && currentTime <= s.end,
+      );
+      if (sentence) {
+        activeSentence = sentence;
+        break;
+      }
+    }
+
+    // Binary search for active word
+    const words = transcript.words;
+    let activeWordIndex: number | null = null;
+    let lo = 0;
+    let hi = words.length - 1;
+    while (lo <= hi) {
+      const mid = (lo + hi) >>> 1;
+      if (currentTime < words[mid].start) hi = mid - 1;
+      else if (currentTime > words[mid].end) lo = mid + 1;
+      else {
+        activeWordIndex = mid;
+        break;
+      }
+    }
+
+    return { activeSentence, activeWordIndex };
+  }, [transcript, currentTime]);
+}