feat: generate transcript and event-driven particle processing

This generates the transcript and shows the caption experience on the
client side for media particles. It also simplifies other side effects
that we must perform such as updating the `last_child_created_at` field
for stream and container particles.

js/src/api/types.ts

@@ -91,11 +91,37 @@ export const FolderPropertiesSchema = z.object({
 });
 export type FolderProperties = z.infer<typeof FolderPropertiesSchema>;
 
+const TranscriptWordSchema = z.object({
+  word: z.string(),
+  start: z.number(),
+  end: z.number(),
+});
+
+const TranscriptSentenceSchema = z.object({
+  text: z.string(),
+  start: z.number(),
+  end: z.number(),
+});
+
+const TranscriptParagraphSchema = z.object({
+  sentences: z.array(TranscriptSentenceSchema),
+  start: z.number(),
+  end: z.number(),
+});
+
+export const TranscriptSchema = z.object({
+  transcript: z.string(),
+  words: z.array(TranscriptWordSchema),
+  paragraphs: z.array(TranscriptParagraphSchema),
+});
+export type Transcript = z.infer<typeof TranscriptSchema>;
+
 export const MediaPropertiesSchema = z.object({
   object_id: z.string(),
   mime_type: z.string(),
   duration_ms: z.number(),
   size_bytes: z.number(),
+  transcript: TranscriptSchema.optional(),
 });
 export type MediaProperties = z.infer<typeof MediaPropertiesSchema>;
 

js/src/features/particles/media-particle-view.tsx

@@ -1,6 +1,8 @@
 import { useEffect, useRef, useState } from "react";
 import type { Particle } from "@/api/types";
 import { useDownloadUrl } from "@/hooks/use-download-url";
+import { useTranscriptPlayback } from "@/hooks/use-transcript-playback";
+import { TranscriptOverlay } from "@/features/particles/transcript-overlay";
 import { Skeleton } from "@/components/ui/skeleton";
 import { AudioLevelBars } from "@/components/audio/audio-level-bars";
 import { useAudioSource } from "@/components/audio/use-audio-source";
@@ -25,6 +27,13 @@ export function MediaParticleView({
   const videoRef = useRef<HTMLVideoElement>(null);
   const audioRef = useRef<HTMLAudioElement>(null);
   const isAudio = particle.properties.mime_type?.startsWith("audio/");
+  const [currentTime, setCurrentTime] = useState(0);
+
+  const transcript = particle.properties.transcript;
+  const { activeParagraph, activeWordIndex } = useTranscriptPlayback(
+    transcript,
+    currentTime,
+  );
 
   // WORKAROUND: useAudioSource needs an audio element, but audioRef is only set after mount
   const [audioEl, setAudioEl] = useState<HTMLAudioElement | null>(null);
@@ -55,6 +64,20 @@ export function MediaParticleView({
     return <Skeleton className="h-full w-full rounded-none" />;
   }
 
+  const handleTimeUpdate = (e: React.SyntheticEvent<HTMLAudioElement | HTMLVideoElement>) => {
+    const { currentTime: time, duration } = e.currentTarget;
+    setCurrentTime(time);
+    if (duration > 0) onProgress?.(time / duration);
+  };
+
+  const captionOverlay = transcript ? (
+    <TranscriptOverlay
+      transcript={transcript}
+      activeParagraph={activeParagraph}
+      activeWordIndex={activeWordIndex}
+    />
+  ) : null;
+
   if (isAudio) {
     return (
       <div className="relative flex h-full w-full items-center justify-center bg-black/90">
@@ -67,10 +90,7 @@ export function MediaParticleView({
           src={url}
           autoPlay
           onEnded={onEnded}
-          onTimeUpdate={(e) => {
-            const { currentTime, duration } = e.currentTarget;
-            if (duration > 0) onProgress?.(currentTime / duration);
-          }}
+          onTimeUpdate={handleTimeUpdate}
         />
 
         {audioSource && (
@@ -78,6 +98,8 @@ export function MediaParticleView({
             <AudioLevelBars sourceNode={audioSource.sourceNode} />
           </div>
         )}
+
+        {captionOverlay}
       </div>
     );
   }
@@ -90,12 +112,11 @@ export function MediaParticleView({
         autoPlay
         playsInline
         onEnded={onEnded}
-        onTimeUpdate={(e) => {
-          const { currentTime, duration } = e.currentTarget;
-          if (duration > 0) onProgress?.(currentTime / duration);
-        }}
+        onTimeUpdate={handleTimeUpdate}
         className="h-full w-full object-cover"
       />
+
+      {captionOverlay}
     </div>
   );
 }

js/src/features/particles/transcript-overlay.tsx

@@ -0,0 +1,53 @@
+import { useMemo } from "react";
+import type { Transcript } from "@/api/types";
+
+interface TranscriptOverlayProps {
+  transcript: Transcript;
+  activeParagraph: Transcript["paragraphs"][number] | null;
+  activeWordIndex: number | null;
+}
+
+export function TranscriptOverlay({
+  transcript,
+  activeParagraph,
+  activeWordIndex,
+}: TranscriptOverlayProps) {
+  // Find the words that belong to the active paragraph by time range
+  const paragraphWords = useMemo(() => {
+    if (!activeParagraph) return [];
+    return transcript.words.filter(
+      (w) => w.start >= activeParagraph.start && w.end <= activeParagraph.end,
+    );
+  }, [transcript.words, activeParagraph]);
+
+  if (!activeParagraph || paragraphWords.length === 0) return null;
+
+  // The active word from the flat array — find it by index to compare
+  const activeWord =
+    activeWordIndex !== null ? transcript.words[activeWordIndex] : null;
+
+  return (
+    <div className="absolute bottom-10 left-0 right-0 flex justify-center px-6 py-4 max-w-md">
+      <p className="rounded-lg bg-black/20 px-5 py-3 text-base leading-relaxed backdrop-blur-sm text-lg text-left">
+        {paragraphWords.map((word, i) => {
+          const isSpoken =
+            activeWord !== null && word.start <= activeWord.end;
+
+          return (
+            <span
+              key={`${word.start}-${i}`}
+              className={
+                isSpoken
+                  ? "text-white font-medium transition-colors duration-150"
+                  : "text-white/40 transition-colors duration-150"
+              }
+            >
+              {i > 0 ? " " : ""}
+              {word.word}
+            </span>
+          );
+        })}
+      </p>
+    </div>
+  );
+}

js/src/hooks/use-create-particle.ts

@@ -1,5 +1,5 @@
 import { useMutation } from "@tanstack/react-query";
-import { createParticle, updateStreamLastChildAt } from "@/lib/firestore-particles";
+import { createParticle } from "@/lib/firestore-particles";
 import type { ParticleType, ParticlePropertiesMap } from "@/api/types";
 import { particlePath, ParticlePath, toFirestoreChildrenPath, toFirestoreDocPath } from "@/lib/particle-path";
 
@@ -15,16 +15,12 @@ export function useCreateParticle() {
   return useMutation({
     mutationFn: async (params: CreateParticleParams) => {
       const collectionPath = toFirestoreChildrenPath(params.path);
-      const result = await createParticle(
+      return await createParticle(
         collectionPath,
         params.type,
         params.properties,
         params.createdByHumanId,
       );
-
-      const streamDocPath = toFirestoreDocPath(params.path);
-      await updateStreamLastChildAt(streamDocPath);
-      return result;
     }
   });
 }

js/src/hooks/use-transcript-playback.ts

@@ -0,0 +1,40 @@
+import { useMemo } from "react";
+import type { Transcript } from "@/api/types";
+
+interface TranscriptPlaybackState {
+  /** The paragraph currently being spoken, or null if before/after speech */
+  activeParagraph: Transcript["paragraphs"][number] | null;
+  /** Index of the active word within the transcript's flat words array */
+  activeWordIndex: number | null;
+}
+
+export function useTranscriptPlayback(
+  transcript: Transcript | undefined,
+  currentTime: number,
+): TranscriptPlaybackState {
+  return useMemo(() => {
+    if (!transcript) return { activeParagraph: null, activeWordIndex: null };
+
+    const activeParagraph =
+      transcript.paragraphs.find(
+        (p) => currentTime >= p.start && currentTime <= p.end,
+      ) ?? null;
+
+    // Binary search for active word
+    const words = transcript.words;
+    let activeWordIndex: number | null = null;
+    let lo = 0;
+    let hi = words.length - 1;
+    while (lo <= hi) {
+      const mid = (lo + hi) >>> 1;
+      if (currentTime < words[mid].start) hi = mid - 1;
+      else if (currentTime > words[mid].end) lo = mid + 1;
+      else {
+        activeWordIndex = mid;
+        break;
+      }
+    }
+
+    return { activeParagraph, activeWordIndex };
+  }, [transcript, currentTime]);
+}

js/src/lib/firestore-particles.ts

@@ -263,16 +263,6 @@ export async function updateParticle(
   });
 }
 
-export async function updateStreamLastChildAt(
-  docPath: string,
-): Promise<void> {
-  const particleRef = typedDoc(docPath);
-  await updateDoc(particleRef, {
-    last_child_created_at: serverTimestamp(),
-    updated_at: serverTimestamp(),
-  });
-}
-
 export async function updateStreamPlaybackMarker(
   docPath: string,
   humanId: string,