From cbb3f50a254b3019a28e942ce1383ff8932e33ba Mon Sep 17 00:00:00 2001 From: talksik Date: Sat, 21 Feb 2026 13:05:02 -0800 Subject: [PATCH] fix: audio bars and upload process --- CLAUDE.md | 3 + js/package.json | 3 +- js/src/api/types.ts | 16 ++-- js/src/components/audio/audio-level-bars.tsx | 76 +++++++++++++++++ js/src/components/audio/use-audio-source.ts | 63 ++++++++++++++ .../features/playback/media-particle-view.tsx | 38 ++++++++- .../features/recording/recording-overlay.tsx | 85 +++++-------------- js/src/features/recording/use-recorder.ts | 20 +++-- js/src/pages/stream-player-page.tsx | 30 ++----- 9 files changed, 226 insertions(+), 108 deletions(-) create mode 100644 js/src/components/audio/audio-level-bars.tsx create mode 100644 js/src/components/audio/use-audio-source.ts diff --git a/CLAUDE.md b/CLAUDE.md index 96948a3..b66a1d5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,3 +23,6 @@ Whenever possible, we should use the design system components. If we need to add When adding a feature on the client side, make sure that the api actually supports it by just checking orion implementation all the way through. IF you find that the API is poorly designed, please suggest changes to improve the client experience. + +## Verification +Check using `yarn compile` which lives in the package.json as a script. diff --git a/js/package.json b/js/package.json index d4f05b8..81599cd 100644 --- a/js/package.json +++ b/js/package.json @@ -10,7 +10,8 @@ "package": "electron-forge package", "make": "electron-forge make", "publish": "electron-forge publish", - "lint": "eslint --ext .ts,.tsx ." + "lint": "eslint --ext .ts,.tsx .", + "compile": "npx tsc --noEmit 2>&1 | grep -E '^src/'" }, "keywords": [], "author": { diff --git a/js/src/api/types.ts b/js/src/api/types.ts index 96a2ad5..e0eb6fa 100644 --- a/js/src/api/types.ts +++ b/js/src/api/types.ts @@ -115,22 +115,16 @@ export interface NetworkWithStreams extends Network { // --- Depot types --- export interface PrepareUploadRequest { - file_name: string; + network_id: string; + name: string; content_type: string; - size_bytes: number; + content_length: number; } export interface PrepareUploadResponse { - object: DepotObject; + object_id: string; upload_url: string; -} - -export interface DepotObject { - id: string; - status: string; - content_type: string; - size_bytes: number; - created_at: string; + upload_headers: Record; } // --- Stream mutation types --- diff --git a/js/src/components/audio/audio-level-bars.tsx b/js/src/components/audio/audio-level-bars.tsx new file mode 100644 index 0000000..e37289f --- /dev/null +++ b/js/src/components/audio/audio-level-bars.tsx @@ -0,0 +1,76 @@ +import { useEffect, useRef, useState } from "react"; + +interface AudioLevelBarsProps { + sourceNode: AudioNode; +} + +/** + * 3-bar VU meter that visualizes audio levels from any AudioNode source. + * Works with both live MediaStream sources and MediaElement sources. + */ +export function AudioLevelBars({ sourceNode }: AudioLevelBarsProps) { + const [levels, setLevels] = useState([0, 0, 0]); + const rafRef = useRef(0); + + useEffect(() => { + const ctx = sourceNode.context as AudioContext; + const analyser = ctx.createAnalyser(); + analyser.fftSize = 256; + sourceNode.connect(analyser); + + // Connect to destination via silent gain node — without this, + // Chromium suspends processing on disconnected audio graphs. + const silentGain = ctx.createGain(); + silentGain.gain.value = 0; + analyser.connect(silentGain); + silentGain.connect(ctx.destination); + + const dataArray = new Uint8Array(analyser.frequencyBinCount); + + function tick() { + analyser.getByteTimeDomainData(dataArray); + + // Compute RMS of waveform (128 = silence baseline) + let sumSquares = 0; + for (let i = 0; i < dataArray.length; i++) { + const normalized = (dataArray[i] - 128) / 128; + sumSquares += normalized * normalized; + } + const rms = Math.sqrt(sumSquares / dataArray.length); + + // VU meter: 3 bars with staggered thresholds + // Typical speech RMS is ~0.02-0.15 from time-domain data + const bar0 = Math.min(1, rms * 10); + const bar1 = Math.max(0, Math.min(1, (rms - 0.02) * 8)); + const bar2 = Math.max(0, Math.min(1, (rms - 0.06) * 6)); + setLevels([bar0, bar1, bar2]); + + rafRef.current = requestAnimationFrame(tick); + } + + rafRef.current = requestAnimationFrame(tick); + + return () => { + cancelAnimationFrame(rafRef.current); + try { + sourceNode.disconnect(analyser); + analyser.disconnect(silentGain); + silentGain.disconnect(ctx.destination); + } catch { + // Nodes may already be disconnected + } + }; + }, [sourceNode]); + + return ( +
+ {levels.map((level, i) => ( +
+ ))} +
+ ); +} diff --git a/js/src/components/audio/use-audio-source.ts b/js/src/components/audio/use-audio-source.ts new file mode 100644 index 0000000..7c4071b --- /dev/null +++ b/js/src/components/audio/use-audio-source.ts @@ -0,0 +1,63 @@ +import { useEffect, useRef, useState } from "react"; + +interface AudioSource { + sourceNode: AudioNode; + ctx: AudioContext; +} + +/** + * Creates an AudioContext and source node from either a MediaStream (live recording) + * or an HTMLAudioElement (review playback). + * + * Important: `createMediaElementSource` can only be called once per element, + * so we cache the source per element instance. + */ +export function useAudioSource( + source: MediaStream | HTMLAudioElement | null, +): AudioSource | null { + const [audioSource, setAudioSource] = useState(null); + const elementSourceCache = useRef< + WeakMap + >(new WeakMap()); + + useEffect(() => { + if (!source) { + setAudioSource(null); + return; + } + + if (source instanceof MediaStream) { + const ctx = new AudioContext(); + ctx.resume(); + const sourceNode = ctx.createMediaStreamSource(source); + setAudioSource({ sourceNode, ctx }); + + return () => { + ctx.close(); + }; + } + + // HTMLAudioElement — createMediaElementSource can only be called once per element + const cached = elementSourceCache.current.get(source); + if (cached) { + cached.ctx.resume(); + setAudioSource(cached); + return; + } + + const ctx = new AudioContext(); + ctx.resume(); + const sourceNode = ctx.createMediaElementSource(source); + // Connect element source to destination so audio is still audible + sourceNode.connect(ctx.destination); + elementSourceCache.current.set(source, { sourceNode, ctx }); + setAudioSource({ sourceNode, ctx }); + + return () => { + ctx.close(); + elementSourceCache.current.delete(source); + }; + }, [source]); + + return audioSource; +} diff --git a/js/src/features/playback/media-particle-view.tsx b/js/src/features/playback/media-particle-view.tsx index 6193c32..9cb4eaa 100644 --- a/js/src/features/playback/media-particle-view.tsx +++ b/js/src/features/playback/media-particle-view.tsx @@ -3,12 +3,21 @@ import type { MediaParticleData, StreamParticle } from "@/api/types"; import { apiClient } from "@/api/client"; import { usePlaybackStore } from "@/stores/playback-store"; import { Skeleton } from "@/components/ui/skeleton"; +import { AudioLevelBars } from "@/components/audio/audio-level-bars"; +import { useAudioSource } from "@/components/audio/use-audio-source"; interface MediaParticleViewProps { particle: StreamParticle; onEnded: () => void; } +function formatTime(ms: number): string { + const totalSeconds = Math.floor(ms / 1000); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes}:${seconds.toString().padStart(2, "0")}`; +} + export function MediaParticleView({ particle, onEnded, @@ -23,6 +32,10 @@ export function MediaParticleView({ const videoRef = useRef(null); const audioRef = useRef(null); + const [audioEl, setAudioEl] = useState(null); + const [currentTimeMs, setCurrentTimeMs] = useState(0); + + const audioSource = useAudioSource(audioEl); useEffect(() => { if (cachedUrl) { @@ -75,8 +88,29 @@ export function MediaParticleView({ if (isAudio) { return ( -
-
- {/* End of stream overlay */} - {status === "ended" && ( -
-
-

End of stream

- -
-
- )} - {/* Text compose overlay */} {composingText && streamId && (