From 3051b1558f32d1eb4c122c8581c02dff6323d60c Mon Sep 17 00:00:00 2001 From: talksik Date: Thu, 30 Apr 2026 17:14:04 -0700 Subject: [PATCH] feat: captions on mobile Closes #199 --- .../stream-view/MediaParticleView.tsx | 39 +++++++ .../src/features/stream-view/StreamView.tsx | 17 +-- .../stream-view/TranscriptOverlay.tsx | 102 ++++++++++++++++++ .../src/hooks/use-transcript-playback.ts | 49 +++++++++ 4 files changed, 200 insertions(+), 7 deletions(-) create mode 100644 js/mobile/src/features/stream-view/TranscriptOverlay.tsx create mode 100644 js/mobile/src/hooks/use-transcript-playback.ts diff --git a/js/mobile/src/features/stream-view/MediaParticleView.tsx b/js/mobile/src/features/stream-view/MediaParticleView.tsx index 9e8aeb8..e10955a 100644 --- a/js/mobile/src/features/stream-view/MediaParticleView.tsx +++ b/js/mobile/src/features/stream-view/MediaParticleView.tsx @@ -7,6 +7,9 @@ import type { Particle } from "@/api/types"; import { apiClient } from "@/api/client"; import { logError } from "@/lib/errors"; import { useEvent } from "@/hooks/use-event"; +import { useTranscriptPlayback } from "@/hooks/use-transcript-playback"; +import { TranscriptOverlay } from "./TranscriptOverlay"; +import { useStreamSafeArea } from "./stream-safe-area"; type MediaParticle = Extract; @@ -90,6 +93,7 @@ function PlayableMediaView({ }) { const [sourceUri, setSourceUri] = useState(null); const [resolveError, setResolveError] = useState(null); + const [currentTime, setCurrentTime] = useState(0); // Fetch the signed download URL once per active object. Orion URLs are // time-limited — we treat the URL as one-shot for this view's lifetime. @@ -99,6 +103,7 @@ function PlayableMediaView({ let cancelled = false; setSourceUri(null); setResolveError(null); + setCurrentTime(0); apiClient .getParticleDownloadUrl(activeObjectId) .then((url) => { @@ -143,6 +148,24 @@ function PlayableMediaView({ } }); + // Drive caption highlighting from the player's own timeUpdate cadence + // (timeUpdateEventInterval = 0.15s above). Pausing halts the events, which + // naturally freezes the active word/sentence — no extra plumbing needed. + useEventListener(player, "timeUpdate", ({ currentTime: t }) => { + setCurrentTime(t); + }); + + const transcript = particle.properties.transcript; + const { activeSentence, activeWordIndex } = useTranscriptPlayback( + transcript, + currentTime, + ); + const safeArea = useStreamSafeArea(); + // Sit just above the compose dock. Pills moved to the top of the screen so + // the only thing this offset has to clear is the dock itself plus a small + // gap — captions can ride lower than they did before. + const captionBottomOffset = safeArea.bottom; + const onEndedStable = useEvent(onEnded); const onProgressStable = useEvent(onProgress); @@ -208,6 +231,14 @@ function PlayableMediaView({ Voice message + {transcript ? ( + + ) : null} ); } @@ -225,6 +256,14 @@ function PlayableMediaView({ allowsFullscreen={false} allowsPictureInPicture={false} /> + {transcript ? ( + + ) : null} ); } diff --git a/js/mobile/src/features/stream-view/StreamView.tsx b/js/mobile/src/features/stream-view/StreamView.tsx index c35d364..4750c29 100644 --- a/js/mobile/src/features/stream-view/StreamView.tsx +++ b/js/mobile/src/features/stream-view/StreamView.tsx @@ -370,10 +370,11 @@ function StreamViewInner({ streamParticle, path, onExit }: StreamViewProps) { const exitRemainingMs = useExitCountdown(status, paused, exit); // Chrome reservations: top = safe-area + segmented bar (3) + gap (12) + - // metadata row (~38) + breathing room (12). Bottom = safe-area + room for - // pause / countdown pills + the compose dock that lands in this same step. + // metadata row (~38) + breathing room (12). Bottom = safe-area + compose + // dock + breathing room. Pause / countdown pills moved to the top so the + // bottom only reserves space for the compose dock now. const chromeTop = insets.top + 65; - const chromeBottom = insets.bottom + 96; + const chromeBottom = insets.bottom + COMPOSE_DOCK_HEIGHT + 14; // --- Render the active particle --- const renderParticle = (particle: Particle) => { @@ -499,12 +500,14 @@ function StreamViewInner({ streamParticle, path, onExit }: StreamViewProps) { - {/* Bottom chrome: paused pill + exit countdown. Sit above the - compose dock so the record button doesn't cover them. */} + {/* Top status pills: paused + exit countdown. Anchored just below + the metadata row (avatar + name ≈ 40px tall, starts at + insets.top + 32) so they share the top chrome real estate + instead of competing with captions at the bottom. */} {paused ? ( diff --git a/js/mobile/src/features/stream-view/TranscriptOverlay.tsx b/js/mobile/src/features/stream-view/TranscriptOverlay.tsx new file mode 100644 index 0000000..c20aedd --- /dev/null +++ b/js/mobile/src/features/stream-view/TranscriptOverlay.tsx @@ -0,0 +1,102 @@ +import { useMemo, useRef } from "react"; +import { Text, View } from "react-native"; +import type { Transcript } from "@/api/types"; + +type Sentence = Transcript["paragraphs"][number]["sentences"][number]; +type Word = Transcript["words"][number]; + +const CHUNK_SIZE = 9; + +function chunkWords(words: Word[]): Word[][] { + const chunks: Word[][] = []; + for (let i = 0; i < words.length; i += CHUNK_SIZE) { + chunks.push(words.slice(i, i + CHUNK_SIZE)); + } + return chunks; +} + +interface TranscriptOverlayProps { + transcript: Transcript; + activeSentence: Sentence | null; + activeWordIndex: number | null; + /** Distance from the bottom of the parent container, in px. */ + bottomOffset: number; +} + +export function TranscriptOverlay({ + transcript, + activeSentence, + activeWordIndex, + bottomOffset, +}: TranscriptOverlayProps) { + const sentenceWords = useMemo(() => { + if (!activeSentence) return []; + return transcript.words.filter( + (w) => w.start >= activeSentence.start && w.end <= activeSentence.end, + ); + }, [transcript.words, activeSentence]); + + const chunks = useMemo(() => chunkWords(sentenceWords), [sentenceWords]); + + const activeWord = + activeWordIndex !== null ? transcript.words[activeWordIndex] : null; + + const lastSpokenWordRef = useRef(null); + if (activeWord) { + lastSpokenWordRef.current = activeWord; + } + const highlightWord = activeWord ?? lastSpokenWordRef.current; + + const lastChunkRef = useRef(null); + + const activeChunk = useMemo(() => { + if (activeWord) { + for (const chunk of chunks) { + if ( + chunk.some( + (w) => w.start === activeWord.start && w.end === activeWord.end, + ) + ) { + lastChunkRef.current = chunk; + return chunk; + } + } + } + if (lastChunkRef.current && chunks.some((c) => c === lastChunkRef.current)) { + return lastChunkRef.current; + } + const fallback = chunks[0] ?? null; + lastChunkRef.current = fallback; + return fallback; + }, [chunks, activeWord]); + + if (!activeSentence || !activeChunk || activeChunk.length === 0) return null; + + return ( + + + + {activeChunk.map((word, i) => { + const isSpoken = + highlightWord !== null && word.start <= highlightWord.end; + return ( + + {i > 0 ? " " : ""} + {word.word} + + ); + })} + + + + ); +} diff --git a/js/mobile/src/hooks/use-transcript-playback.ts b/js/mobile/src/hooks/use-transcript-playback.ts new file mode 100644 index 0000000..b047362 --- /dev/null +++ b/js/mobile/src/hooks/use-transcript-playback.ts @@ -0,0 +1,49 @@ +import { useMemo } from "react"; +import type { Transcript } from "@/api/types"; + +type Sentence = Transcript["paragraphs"][number]["sentences"][number]; + +interface TranscriptPlaybackState { + /** The sentence currently being spoken, or null if between sentences */ + activeSentence: Sentence | null; + /** Index of the active word within the transcript's flat words array */ + activeWordIndex: number | null; +} + +export function useTranscriptPlayback( + transcript: Transcript | undefined, + currentTime: number, +): TranscriptPlaybackState { + return useMemo(() => { + if (!transcript) return { activeSentence: null, activeWordIndex: null }; + + // Find the active sentence across all paragraphs + let activeSentence: Sentence | null = null; + for (const paragraph of transcript.paragraphs) { + const sentence = paragraph.sentences.find( + (s) => currentTime >= s.start && currentTime <= s.end, + ); + if (sentence) { + activeSentence = sentence; + break; + } + } + + // Binary search for active word + const words = transcript.words; + let activeWordIndex: number | null = null; + let lo = 0; + let hi = words.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >>> 1; + if (currentTime < words[mid].start) hi = mid - 1; + else if (currentTime > words[mid].end) lo = mid + 1; + else { + activeWordIndex = mid; + break; + } + } + + return { activeSentence, activeWordIndex }; + }, [transcript, currentTime]); +}