FlowyLabs/llink
1
0
Fork 0
Code Issues 71 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: audio bars and upload process

Browse Source
This commit is contained in:
talksik
2026-02-21 13:05:02 -08:00
parent 8546260493
commit cbb3f50a25
9 changed files with 226 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/src/api/types.ts
+5 -11
View File
@@ -115,22 +115,16 @@ export interface NetworkWithStreams extends Network {
// --- Depot types ---
export interface PrepareUploadRequest {
file_name: string;
network_id: string;
name: string;
content_type: string;
size_bytes: number;
content_length: number;
}
export interface PrepareUploadResponse {
object: DepotObject;
object_id: string;
upload_url: string;
}
export interface DepotObject {
id: string;
status: string;
content_type: string;
size_bytes: number;
created_at: string;
upload_headers: Record<string, string>;
}
// --- Stream mutation types ---
js/src/components/audio/audio-level-bars.tsx
+76
View File
@@ -0,0 +1,76 @@
import { useEffect, useRef, useState } from "react";
interface AudioLevelBarsProps {
sourceNode: AudioNode;
}
/**
* 3-bar VU meter that visualizes audio levels from any AudioNode source.
* Works with both live MediaStream sources and MediaElement sources.
*/
export function AudioLevelBars({ sourceNode }: AudioLevelBarsProps) {
const [levels, setLevels] = useState([0, 0, 0]);
const rafRef = useRef<number>(0);
useEffect(() => {
const ctx = sourceNode.context as AudioContext;
const analyser = ctx.createAnalyser();
analyser.fftSize = 256;
sourceNode.connect(analyser);
// Connect to destination via silent gain node — without this,
// Chromium suspends processing on disconnected audio graphs.
const silentGain = ctx.createGain();
silentGain.gain.value = 0;
analyser.connect(silentGain);
silentGain.connect(ctx.destination);
const dataArray = new Uint8Array(analyser.frequencyBinCount);
function tick() {
analyser.getByteTimeDomainData(dataArray);
// Compute RMS of waveform (128 = silence baseline)
let sumSquares = 0;
for (let i = 0; i < dataArray.length; i++) {
const normalized = (dataArray[i] - 128) / 128;
sumSquares += normalized * normalized;
}
const rms = Math.sqrt(sumSquares / dataArray.length);
// VU meter: 3 bars with staggered thresholds
// Typical speech RMS is ~0.02-0.15 from time-domain data
const bar0 = Math.min(1, rms * 10);
const bar1 = Math.max(0, Math.min(1, (rms - 0.02) * 8));
const bar2 = Math.max(0, Math.min(1, (rms - 0.06) * 6));
setLevels([bar0, bar1, bar2]);
rafRef.current = requestAnimationFrame(tick);
}
rafRef.current = requestAnimationFrame(tick);
return () => {
cancelAnimationFrame(rafRef.current);
try {
sourceNode.disconnect(analyser);
analyser.disconnect(silentGain);
silentGain.disconnect(ctx.destination);
} catch {
// Nodes may already be disconnected
}
};
}, [sourceNode]);
return (
<div className="flex items-end gap-1.5">
{levels.map((level, i) => (
<div
key={i}
className="w-1.5 rounded-full bg-green-400 transition-all duration-75"
style={{ height: `${Math.max(6, level * 48)}px` }}
/>
))}
</div>
);
}
js/src/components/audio/use-audio-source.ts
+63
View File
@@ -0,0 +1,63 @@
import { useEffect, useRef, useState } from "react";
interface AudioSource {
sourceNode: AudioNode;
ctx: AudioContext;
}
/**
* Creates an AudioContext and source node from either a MediaStream (live recording)
* or an HTMLAudioElement (review playback).
*
* Important: `createMediaElementSource` can only be called once per element,
* so we cache the source per element instance.
*/
export function useAudioSource(
source: MediaStream | HTMLAudioElement | null,
): AudioSource | null {
const [audioSource, setAudioSource] = useState<AudioSource | null>(null);
const elementSourceCache = useRef<
WeakMap<HTMLAudioElement, { sourceNode: MediaElementAudioSourceNode; ctx: AudioContext }>
>(new WeakMap());
useEffect(() => {
if (!source) {
setAudioSource(null);
return;
}
if (source instanceof MediaStream) {
const ctx = new AudioContext();
ctx.resume();
const sourceNode = ctx.createMediaStreamSource(source);
setAudioSource({ sourceNode, ctx });
return () => {
ctx.close();
};
}
// HTMLAudioElement — createMediaElementSource can only be called once per element
const cached = elementSourceCache.current.get(source);
if (cached) {
cached.ctx.resume();
setAudioSource(cached);
return;
}
const ctx = new AudioContext();
ctx.resume();
const sourceNode = ctx.createMediaElementSource(source);
// Connect element source to destination so audio is still audible
sourceNode.connect(ctx.destination);
elementSourceCache.current.set(source, { sourceNode, ctx });
setAudioSource({ sourceNode, ctx });
return () => {
ctx.close();
elementSourceCache.current.delete(source);
};
}, [source]);
return audioSource;
}
js/src/features/playback/media-particle-view.tsx
+36 -2
View File
@@ -3,12 +3,21 @@ import type { MediaParticleData, StreamParticle } from "@/api/types";
import { apiClient } from "@/api/client";
import { usePlaybackStore } from "@/stores/playback-store";
import { Skeleton } from "@/components/ui/skeleton";
import { AudioLevelBars } from "@/components/audio/audio-level-bars";
import { useAudioSource } from "@/components/audio/use-audio-source";
interface MediaParticleViewProps {
particle: StreamParticle;
onEnded: () => void;
}
function formatTime(ms: number): string {
const totalSeconds = Math.floor(ms / 1000);
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
}
export function MediaParticleView({
particle,
onEnded,
@@ -23,6 +32,10 @@ export function MediaParticleView({
const videoRef = useRef<HTMLVideoElement>(null);
const audioRef = useRef<HTMLAudioElement>(null);
const [audioEl, setAudioEl] = useState<HTMLAudioElement | null>(null);
const [currentTimeMs, setCurrentTimeMs] = useState(0);
const audioSource = useAudioSource(audioEl);
useEffect(() => {
if (cachedUrl) {
@@ -75,8 +88,29 @@ export function MediaParticleView({
if (isAudio) {
return (
<div className="flex h-full w-full items-center justify-center">
<audio ref={audioRef} src={url} autoPlay onEnded={onEnded} controls />
<div className="relative flex h-full w-full items-center justify-center bg-black/90">
<audio
ref={(el) => {
audioRef.current = el;
setAudioEl(el);
}}
src={url}
autoPlay
onEnded={onEnded}
onTimeUpdate={(e) => {
setCurrentTimeMs(e.currentTarget.currentTime * 1000);
}}
/>
{audioSource && (
<AudioLevelBars sourceNode={audioSource.sourceNode} />
)}
<div className="absolute top-3 right-3 rounded-full bg-black/30 px-2.5 py-1 backdrop-blur-sm">
<span className="font-mono text-xs text-white/80">
{formatTime(currentTimeMs)} / {formatTime(data.duration_ms)}
</span>
</div>
</div>
);
}
js/src/features/recording/recording-overlay.tsx
+22 -63
View File
@@ -1,69 +1,12 @@
import { useEffect, useRef, useState } from "react";
import { useRecordingStore } from "@/stores/recording-store";
import { AudioLevelBars } from "@/components/audio/audio-level-bars";
import { useAudioSource } from "@/components/audio/use-audio-source";
interface RecordingOverlayProps {
onClose: () => void;
}
function AudioLevelBars({ mediaStream }: { mediaStream: MediaStream }) {
const audioRef = useRef<{ analyser: AnalyserNode; ctx: AudioContext } | null>(
null,
);
const [levels, setLevels] = useState([0, 0, 0]);
const rafRef = useRef<number>(0);
useEffect(() => {
const ctx = new AudioContext();
ctx.resume();
const source = ctx.createMediaStreamSource(mediaStream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 256;
source.connect(analyser);
audioRef.current = { analyser, ctx };
const dataArray = new Uint8Array(analyser.frequencyBinCount);
function tick() {
analyser.getByteTimeDomainData(dataArray);
// Compute RMS of waveform (128 = silence baseline)
let sumSquares = 0;
for (let i = 0; i < dataArray.length; i++) {
const normalized = (dataArray[i] - 128) / 128;
sumSquares += normalized * normalized;
}
const rms = Math.sqrt(sumSquares / dataArray.length);
// VU meter: 3 bars with staggered thresholds
const bar0 = Math.min(1, rms * 3);
const bar1 = Math.max(0, Math.min(1, (rms - 0.1) * 3));
const bar2 = Math.max(0, Math.min(1, (rms - 0.25) * 3));
setLevels([bar0, bar1, bar2]);
rafRef.current = requestAnimationFrame(tick);
}
rafRef.current = requestAnimationFrame(tick);
return () => {
cancelAnimationFrame(rafRef.current);
ctx.close();
};
}, [mediaStream]);
return (
<div className="flex items-end gap-1.5">
{levels.map((level, i) => (
<div
key={i}
className="w-1.5 rounded-full bg-green-400 transition-all duration-75"
style={{ height: `${Math.max(6, level * 48)}px` }}
/>
))}
</div>
);
}
function RecordingTimer() {
const [elapsed, setElapsed] = useState(0);
@@ -95,6 +38,9 @@ function ReviewPlayback({
}) {
const urlRef = useRef<string | null>(null);
const [objectUrl, setObjectUrl] = useState<string | null>(null);
const audioElRef = useRef<HTMLAudioElement | null>(null);
const [audioEl, setAudioEl] = useState<HTMLAudioElement | null>(null);
const audioSource = useAudioSource(isVideo ? null : audioEl);
useEffect(() => {
const url = URL.createObjectURL(blob);
@@ -123,8 +69,20 @@ function ReviewPlayback({
return (
<div className="flex flex-col items-center gap-3">
<audio src={objectUrl} autoPlay loop />
<span className="text-sm text-white/60">Playing back audio...</span>
<audio
ref={(el) => {
audioElRef.current = el;
setAudioEl(el);
}}
src={objectUrl}
autoPlay
loop
/>
{audioSource ? (
<AudioLevelBars sourceNode={audioSource.sourceNode} />
) : (
<span className="text-sm text-white/60">Playing back audio...</span>
)}
</div>
);
}
@@ -136,6 +94,7 @@ export function RecordingOverlay({ onClose }: RecordingOverlayProps) {
const reviewBlob = useRecordingStore((s) => s.reviewBlob);
const videoRef = useRef<HTMLVideoElement>(null);
const hasBeenActiveRef = useRef(false);
const recordingAudioSource = useAudioSource(mediaStream ?? null);
// Track whether we've entered an active state at least once
if (
@@ -227,9 +186,9 @@ export function RecordingOverlay({ onClose }: RecordingOverlayProps) {
</div>
{/* Center: audio level bars (recording with active stream) */}
{isRecording && mediaStream && (
{isRecording && recordingAudioSource && (
<div className="z-10">
<AudioLevelBars mediaStream={mediaStream} />
<AudioLevelBars sourceNode={recordingAudioSource.sourceNode} />
</div>
)}
js/src/features/recording/use-recorder.ts
+12 -8
View File
@@ -20,7 +20,10 @@ function getMediaMime(mode: "video" | "audio"): string {
return VIDEO_FALLBACK_MIME;
}
export function useRecorder(streamId: string | null) {
export function useRecorder(
streamId: string | null,
networkId: string | null,
) {
const recorderRef = useRef<MediaRecorder | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const chunksRef = useRef<Blob[]>([]);
@@ -44,7 +47,7 @@ export function useRecorder(streamId: string | null) {
}, [setMediaStream]);
const confirmSend = useCallback(async () => {
if (!streamId) return;
if (!streamId || !networkId) return;
const { reviewBlob, reviewDurationMs } = useRecordingStore.getState();
if (!reviewBlob) return;
@@ -55,10 +58,11 @@ export function useRecorder(streamId: string | null) {
const mimeType = reviewBlob.type || VIDEO_FALLBACK_MIME;
const fileName = `recording-${Date.now()}.webm`;
const { object, upload_url } = await apiClient.prepareUpload({
file_name: fileName,
const { object_id, upload_url } = await apiClient.prepareUpload({
network_id: networkId,
name: fileName,
content_type: mimeType,
size_bytes: reviewBlob.size,
content_length: reviewBlob.size,
});
await fetch(upload_url, {
@@ -67,12 +71,12 @@ export function useRecorder(streamId: string | null) {
body: reviewBlob,
});
await apiClient.confirmUpload(object.id);
await apiClient.confirmUpload(object_id);
const particle = await apiClient.createStreamParticle(streamId, {
type: "media",
data: {
object_id: object.id,
object_id,
duration_ms: reviewDurationMs,
mime_type: mimeType,
},
@@ -91,7 +95,7 @@ export function useRecorder(streamId: string | null) {
} catch (err) {
setError(err instanceof Error ? err.message : "Upload failed");
}
}, [streamId, setStatus, setError, resetRecording, addParticleToStream]);
}, [streamId, networkId, setStatus, setError, resetRecording, addParticleToStream]);
const startRecording = useCallback(async () => {
const currentStatus = useRecordingStore.getState().status;
js/src/pages/stream-player-page.tsx
+7 -23
View File
@@ -21,7 +21,6 @@ export function StreamPlayerPage() {
const particles = usePlaybackStore((s) => s.particles);
const currentIndex = usePlaybackStore((s) => s.currentIndex);
const status = usePlaybackStore((s) => s.status);
const initStream = usePlaybackStore((s) => s.initStream);
const next = usePlaybackStore((s) => s.next);
const prev = usePlaybackStore((s) => s.prev);
@@ -30,13 +29,14 @@ export function StreamPlayerPage() {
const pause = usePlaybackStore((s) => s.pause);
const resume = usePlaybackStore((s) => s.resume);
const { startRecording, stopRecording, cancelRecording, confirmSend } =
useRecorder(streamId ?? null);
// Find the stream and its parent network
const networkWithStream = networks.find((n) =>
n.streams.some((s) => s.id === streamId),
);
const stream = networkWithStream?.streams.find((s) => s.id === streamId);
// Find the stream across all networks
const stream = networks
.flatMap((n) => n.streams)
.find((s) => s.id === streamId);
const { startRecording, stopRecording, cancelRecording, confirmSend } =
useRecorder(streamId ?? null, networkWithStream?.id ?? null);
useEffect(() => {
if (!stream) return;
@@ -246,22 +246,6 @@ export function StreamPlayerPage() {
</div>
</div>
{/* End of stream overlay */}
{status === "ended" && (
<div className="absolute inset-0 z-20 flex items-center justify-center bg-black/80">
<div className="flex flex-col items-center gap-4">
<p className="text-sm text-white">End of stream</p>
<Button
variant="outline"
size="sm"
onClick={() => navigate("/")}
>
Back to streams
</Button>
</div>
</div>
)}
{/* Text compose overlay */}
{composingText && streamId && (
<TextComposeOverlay