feat: generate transcript and event-driven particle processing
This generates the transcript and shows the caption experience on the client side for media particles. It also simplifies other side effects that we must perform such as updating the `last_child_created_at` field for stream and container particles.
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import type { Particle } from "@/api/types";
|
||||
import { useDownloadUrl } from "@/hooks/use-download-url";
|
||||
import { useTranscriptPlayback } from "@/hooks/use-transcript-playback";
|
||||
import { TranscriptOverlay } from "@/features/particles/transcript-overlay";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { AudioLevelBars } from "@/components/audio/audio-level-bars";
|
||||
import { useAudioSource } from "@/components/audio/use-audio-source";
|
||||
@@ -25,6 +27,13 @@ export function MediaParticleView({
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const audioRef = useRef<HTMLAudioElement>(null);
|
||||
const isAudio = particle.properties.mime_type?.startsWith("audio/");
|
||||
const [currentTime, setCurrentTime] = useState(0);
|
||||
|
||||
const transcript = particle.properties.transcript;
|
||||
const { activeParagraph, activeWordIndex } = useTranscriptPlayback(
|
||||
transcript,
|
||||
currentTime,
|
||||
);
|
||||
|
||||
// WORKAROUND: useAudioSource needs an audio element, but audioRef is only set after mount
|
||||
const [audioEl, setAudioEl] = useState<HTMLAudioElement | null>(null);
|
||||
@@ -55,6 +64,20 @@ export function MediaParticleView({
|
||||
return <Skeleton className="h-full w-full rounded-none" />;
|
||||
}
|
||||
|
||||
const handleTimeUpdate = (e: React.SyntheticEvent<HTMLAudioElement | HTMLVideoElement>) => {
|
||||
const { currentTime: time, duration } = e.currentTarget;
|
||||
setCurrentTime(time);
|
||||
if (duration > 0) onProgress?.(time / duration);
|
||||
};
|
||||
|
||||
const captionOverlay = transcript ? (
|
||||
<TranscriptOverlay
|
||||
transcript={transcript}
|
||||
activeParagraph={activeParagraph}
|
||||
activeWordIndex={activeWordIndex}
|
||||
/>
|
||||
) : null;
|
||||
|
||||
if (isAudio) {
|
||||
return (
|
||||
<div className="relative flex h-full w-full items-center justify-center bg-black/90">
|
||||
@@ -67,10 +90,7 @@ export function MediaParticleView({
|
||||
src={url}
|
||||
autoPlay
|
||||
onEnded={onEnded}
|
||||
onTimeUpdate={(e) => {
|
||||
const { currentTime, duration } = e.currentTarget;
|
||||
if (duration > 0) onProgress?.(currentTime / duration);
|
||||
}}
|
||||
onTimeUpdate={handleTimeUpdate}
|
||||
/>
|
||||
|
||||
{audioSource && (
|
||||
@@ -78,6 +98,8 @@ export function MediaParticleView({
|
||||
<AudioLevelBars sourceNode={audioSource.sourceNode} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{captionOverlay}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -90,12 +112,11 @@ export function MediaParticleView({
|
||||
autoPlay
|
||||
playsInline
|
||||
onEnded={onEnded}
|
||||
onTimeUpdate={(e) => {
|
||||
const { currentTime, duration } = e.currentTarget;
|
||||
if (duration > 0) onProgress?.(currentTime / duration);
|
||||
}}
|
||||
onTimeUpdate={handleTimeUpdate}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
|
||||
{captionOverlay}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user