feat: generate transcript and event-driven particle processing

This generates the transcript and shows the caption experience on the
client side for media particles. It also simplifies other side effects
that we must perform such as updating the `last_child_created_at` field
for stream and container particles.
This commit is contained in:
talksik
2026-03-25 14:43:47 -07:00
parent 92bbaa11b3
commit 986a389606
17 changed files with 578 additions and 52 deletions
@@ -1,6 +1,8 @@
import { useEffect, useRef, useState } from "react";
import type { Particle } from "@/api/types";
import { useDownloadUrl } from "@/hooks/use-download-url";
import { useTranscriptPlayback } from "@/hooks/use-transcript-playback";
import { TranscriptOverlay } from "@/features/particles/transcript-overlay";
import { Skeleton } from "@/components/ui/skeleton";
import { AudioLevelBars } from "@/components/audio/audio-level-bars";
import { useAudioSource } from "@/components/audio/use-audio-source";
@@ -25,6 +27,13 @@ export function MediaParticleView({
const videoRef = useRef<HTMLVideoElement>(null);
const audioRef = useRef<HTMLAudioElement>(null);
const isAudio = particle.properties.mime_type?.startsWith("audio/");
const [currentTime, setCurrentTime] = useState(0);
const transcript = particle.properties.transcript;
const { activeParagraph, activeWordIndex } = useTranscriptPlayback(
transcript,
currentTime,
);
// WORKAROUND: useAudioSource needs an audio element, but audioRef is only set after mount
const [audioEl, setAudioEl] = useState<HTMLAudioElement | null>(null);
@@ -55,6 +64,20 @@ export function MediaParticleView({
return <Skeleton className="h-full w-full rounded-none" />;
}
const handleTimeUpdate = (e: React.SyntheticEvent<HTMLAudioElement | HTMLVideoElement>) => {
const { currentTime: time, duration } = e.currentTarget;
setCurrentTime(time);
if (duration > 0) onProgress?.(time / duration);
};
const captionOverlay = transcript ? (
<TranscriptOverlay
transcript={transcript}
activeParagraph={activeParagraph}
activeWordIndex={activeWordIndex}
/>
) : null;
if (isAudio) {
return (
<div className="relative flex h-full w-full items-center justify-center bg-black/90">
@@ -67,10 +90,7 @@ export function MediaParticleView({
src={url}
autoPlay
onEnded={onEnded}
onTimeUpdate={(e) => {
const { currentTime, duration } = e.currentTarget;
if (duration > 0) onProgress?.(currentTime / duration);
}}
onTimeUpdate={handleTimeUpdate}
/>
{audioSource && (
@@ -78,6 +98,8 @@ export function MediaParticleView({
<AudioLevelBars sourceNode={audioSource.sourceNode} />
</div>
)}
{captionOverlay}
</div>
);
}
@@ -90,12 +112,11 @@ export function MediaParticleView({
autoPlay
playsInline
onEnded={onEnded}
onTimeUpdate={(e) => {
const { currentTime, duration } = e.currentTarget;
if (duration > 0) onProgress?.(currentTime / duration);
}}
onTimeUpdate={handleTimeUpdate}
className="h-full w-full object-cover"
/>
{captionOverlay}
</div>
);
}