FlowyLabs/llink
1
0
Fork 0
Code Issues 71 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: audio bars and upload process

Browse Source
This commit is contained in:
talksik
2026-02-21 13:05:02 -08:00
parent 8546260493
commit cbb3f50a25
9 changed files with 226 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CLAUDE.md
+3
View File
@@ -23,3 +23,6 @@ Whenever possible, we should use the design system components. If we need to add
When adding a feature on the client side, make sure that the api actually supports it by just checking orion implementation all the way through.
IF you find that the API is poorly designed, please suggest changes to improve the client experience.
## Verification
Check using `yarn compile` which lives in the package.json as a script.
js/package.json
+2 -1
View File
@@ -10,7 +10,8 @@
"package": "electron-forge package",
"make": "electron-forge make",
"publish": "electron-forge publish",
"lint": "eslint --ext .ts,.tsx ."
"lint": "eslint --ext .ts,.tsx .",
"compile": "npx tsc --noEmit 2>&1 | grep -E '^src/'"
},
"keywords": [],
"author": {
js/src/api/types.ts
+5 -11
View File
@@ -115,22 +115,16 @@ export interface NetworkWithStreams extends Network {
// --- Depot types ---
export interface PrepareUploadRequest {
file_name: string;
network_id: string;
name: string;
content_type: string;
size_bytes: number;
content_length: number;
}
export interface PrepareUploadResponse {
object: DepotObject;
object_id: string;
upload_url: string;
}
export interface DepotObject {
id: string;
status: string;
content_type: string;
size_bytes: number;
created_at: string;
upload_headers: Record<string, string>;
}
// --- Stream mutation types ---
js/src/components/audio/audio-level-bars.tsx
+76
View File
@@ -0,0 +1,76 @@
import { useEffect, useRef, useState } from "react";
interface AudioLevelBarsProps {
sourceNode: AudioNode;
}
/**
* 3-bar VU meter that visualizes audio levels from any AudioNode source.
* Works with both live MediaStream sources and MediaElement sources.
*/
export function AudioLevelBars({ sourceNode }: AudioLevelBarsProps) {
const [levels, setLevels] = useState([0, 0, 0]);
const rafRef = useRef<number>(0);
useEffect(() => {
const ctx = sourceNode.context as AudioContext;
const analyser = ctx.createAnalyser();
analyser.fftSize = 256;
sourceNode.connect(analyser);
// Connect to destination via silent gain node — without this,
// Chromium suspends processing on disconnected audio graphs.
const silentGain = ctx.createGain();
silentGain.gain.value = 0;
analyser.connect(silentGain);
silentGain.connect(ctx.destination);
const dataArray = new Uint8Array(analyser.frequencyBinCount);
function tick() {
analyser.getByteTimeDomainData(dataArray);
// Compute RMS of waveform (128 = silence baseline)
let sumSquares = 0;
for (let i = 0; i < dataArray.length; i++) {
const normalized = (dataArray[i] - 128) / 128;
sumSquares += normalized * normalized;
}
const rms = Math.sqrt(sumSquares / dataArray.length);
// VU meter: 3 bars with staggered thresholds
// Typical speech RMS is ~0.02-0.15 from time-domain data
const bar0 = Math.min(1, rms * 10);
const bar1 = Math.max(0, Math.min(1, (rms - 0.02) * 8));
const bar2 = Math.max(0, Math.min(1, (rms - 0.06) * 6));
setLevels([bar0, bar1, bar2]);
rafRef.current = requestAnimationFrame(tick);
}
rafRef.current = requestAnimationFrame(tick);
return () => {
cancelAnimationFrame(rafRef.current);
try {
sourceNode.disconnect(analyser);
analyser.disconnect(silentGain);
silentGain.disconnect(ctx.destination);
} catch {
// Nodes may already be disconnected
}
};
}, [sourceNode]);
return (
<div className="flex items-end gap-1.5">
{levels.map((level, i) => (
<div
key={i}
className="w-1.5 rounded-full bg-green-400 transition-all duration-75"
style={{ height: `${Math.max(6, level * 48)}px` }}
/>
))}
</div>
);
}
js/src/components/audio/use-audio-source.ts
+63
View File
@@ -0,0 +1,63 @@
import { useEffect, useRef, useState } from "react";
interface AudioSource {
sourceNode: AudioNode;
ctx: AudioContext;
}
/**
* Creates an AudioContext and source node from either a MediaStream (live recording)
* or an HTMLAudioElement (review playback).
*
* Important: `createMediaElementSource` can only be called once per element,
* so we cache the source per element instance.
*/
export function useAudioSource(
source: MediaStream | HTMLAudioElement | null,
): AudioSource | null {
const [audioSource, setAudioSource] = useState<AudioSource | null>(null);
const elementSourceCache = useRef<
WeakMap<HTMLAudioElement, { sourceNode: MediaElementAudioSourceNode; ctx: AudioContext }>
>(new WeakMap());
useEffect(() => {
if (!source) {
setAudioSource(null);
return;
}
if (source instanceof MediaStream) {
const ctx = new AudioContext();
ctx.resume();
const sourceNode = ctx.createMediaStreamSource(source);
setAudioSource({ sourceNode, ctx });
return () => {
ctx.close();
};
}
// HTMLAudioElement — createMediaElementSource can only be called once per element
const cached = elementSourceCache.current.get(source);
if (cached) {
cached.ctx.resume();
setAudioSource(cached);
return;
}
const ctx = new AudioContext();
ctx.resume();
const sourceNode = ctx.createMediaElementSource(source);
// Connect element source to destination so audio is still audible
sourceNode.connect(ctx.destination);
elementSourceCache.current.set(source, { sourceNode, ctx });
setAudioSource({ sourceNode, ctx });
return () => {
ctx.close();
elementSourceCache.current.delete(source);
};
}, [source]);
return audioSource;
}
js/src/features/playback/media-particle-view.tsx
+36 -2
View File
@@ -3,12 +3,21 @@ import type { MediaParticleData, StreamParticle } from "@/api/types";
import { apiClient } from "@/api/client";
import { usePlaybackStore } from "@/stores/playback-store";
import { Skeleton } from "@/components/ui/skeleton";
import { AudioLevelBars } from "@/components/audio/audio-level-bars";
import { useAudioSource } from "@/components/audio/use-audio-source";
interface MediaParticleViewProps {
particle: StreamParticle;
onEnded: () => void;
}
function formatTime(ms: number): string {
const totalSeconds = Math.floor(ms / 1000);
const minutes = Math.floor(totalSeconds / 60);
const seconds = totalSeconds % 60;
return `${minutes}:${seconds.toString().padStart(2, "0")}`;
}
export function MediaParticleView({
particle,
onEnded,
@@ -23,6 +32,10 @@ export function MediaParticleView({
const videoRef = useRef<HTMLVideoElement>(null);
const audioRef = useRef<HTMLAudioElement>(null);
const [audioEl, setAudioEl] = useState<HTMLAudioElement | null>(null);
const [currentTimeMs, setCurrentTimeMs] = useState(0);
const audioSource = useAudioSource(audioEl);
useEffect(() => {
if (cachedUrl) {
@@ -75,8 +88,29 @@ export function MediaParticleView({
if (isAudio) {
return (
<div className="flex h-full w-full items-center justify-center">
<audio ref={audioRef} src={url} autoPlay onEnded={onEnded} controls />
<div className="relative flex h-full w-full items-center justify-center bg-black/90">
<audio
ref={(el) => {
audioRef.current = el;
setAudioEl(el);
}}
src={url}
autoPlay
onEnded={onEnded}
onTimeUpdate={(e) => {
setCurrentTimeMs(e.currentTarget.currentTime * 1000);
}}
/>
{audioSource && (
<AudioLevelBars sourceNode={audioSource.sourceNode} />
)}
<div className="absolute top-3 right-3 rounded-full bg-black/30 px-2.5 py-1 backdrop-blur-sm">
<span className="font-mono text-xs text-white/80">
{formatTime(currentTimeMs)} / {formatTime(data.duration_ms)}
</span>
</div>
</div>
);
}
js/src/features/recording/recording-overlay.tsx
+22 -63
View File
@@ -1,69 +1,12 @@
import { useEffect, useRef, useState } from "react";
import { useRecordingStore } from "@/stores/recording-store";
import { AudioLevelBars } from "@/components/audio/audio-level-bars";
import { useAudioSource } from "@/components/audio/use-audio-source";
interface RecordingOverlayProps {
onClose: () => void;
}
function AudioLevelBars({ mediaStream }: { mediaStream: MediaStream }) {
const audioRef = useRef<{ analyser: AnalyserNode; ctx: AudioContext } | null>(
null,
);
const [levels, setLevels] = useState([0, 0, 0]);
const rafRef = useRef<number>(0);
useEffect(() => {
const ctx = new AudioContext();
ctx.resume();
const source = ctx.createMediaStreamSource(mediaStream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 256;
source.connect(analyser);
audioRef.current = { analyser, ctx };
const dataArray = new Uint8Array(analyser.frequencyBinCount);
function tick() {
analyser.getByteTimeDomainData(dataArray);
// Compute RMS of waveform (128 = silence baseline)
let sumSquares = 0;
for (let i = 0; i < dataArray.length; i++) {
const normalized = (dataArray[i] - 128) / 128;
sumSquares += normalized * normalized;
}
const rms = Math.sqrt(sumSquares / dataArray.length);
// VU meter: 3 bars with staggered thresholds
const bar0 = Math.min(1, rms * 3);
const bar1 = Math.max(0, Math.min(1, (rms - 0.1) * 3));
const bar2 = Math.max(0, Math.min(1, (rms - 0.25) * 3));
setLevels([bar0, bar1, bar2]);
rafRef.current = requestAnimationFrame(tick);
}
rafRef.current = requestAnimationFrame(tick);
return () => {
cancelAnimationFrame(rafRef.current);
ctx.close();
};
}, [mediaStream]);
return (
<div className="flex items-end gap-1.5">
{levels.map((level, i) => (
<div
key={i}
className="w-1.5 rounded-full bg-green-400 transition-all duration-75"
style={{ height: `${Math.max(6, level * 48)}px` }}
/>
))}
</div>
);
}
function RecordingTimer() {
const [elapsed, setElapsed] = useState(0);
@@ -95,6 +38,9 @@ function ReviewPlayback({
}) {
const urlRef = useRef<string | null>(null);
const [objectUrl, setObjectUrl] = useState<string | null>(null);
const audioElRef = useRef<HTMLAudioElement | null>(null);
const [audioEl, setAudioEl] = useState<HTMLAudioElement | null>(null);
const audioSource = useAudioSource(isVideo ? null : audioEl);
useEffect(() => {
const url = URL.createObjectURL(blob);
@@ -123,8 +69,20 @@ function ReviewPlayback({
return (
<div className="flex flex-col items-center gap-3">
<audio src={objectUrl} autoPlay loop />
<span className="text-sm text-white/60">Playing back audio...</span>
<audio
ref={(el) => {
audioElRef.current = el;
setAudioEl(el);
}}
src={objectUrl}
autoPlay
loop
/>
{audioSource ? (
<AudioLevelBars sourceNode={audioSource.sourceNode} />
) : (
<span className="text-sm text-white/60">Playing back audio...</span>
)}
</div>
);
}
@@ -136,6 +94,7 @@ export function RecordingOverlay({ onClose }: RecordingOverlayProps) {
const reviewBlob = useRecordingStore((s) => s.reviewBlob);
const videoRef = useRef<HTMLVideoElement>(null);
const hasBeenActiveRef = useRef(false);
const recordingAudioSource = useAudioSource(mediaStream ?? null);
// Track whether we've entered an active state at least once
if (
@@ -227,9 +186,9 @@ export function RecordingOverlay({ onClose }: RecordingOverlayProps) {
</div>
{/* Center: audio level bars (recording with active stream) */}
{isRecording && mediaStream && (
{isRecording && recordingAudioSource && (
<div className="z-10">
<AudioLevelBars mediaStream={mediaStream} />
<AudioLevelBars sourceNode={recordingAudioSource.sourceNode} />
</div>
)}
js/src/features/recording/use-recorder.ts
+12 -8
View File
@@ -20,7 +20,10 @@ function getMediaMime(mode: "video" | "audio"): string {
return VIDEO_FALLBACK_MIME;
}
export function useRecorder(streamId: string | null) {
export function useRecorder(
streamId: string | null,
networkId: string | null,
) {
const recorderRef = useRef<MediaRecorder | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const chunksRef = useRef<Blob[]>([]);
@@ -44,7 +47,7 @@ export function useRecorder(streamId: string | null) {
}, [setMediaStream]);
const confirmSend = useCallback(async () => {
if (!streamId) return;
if (!streamId || !networkId) return;
const { reviewBlob, reviewDurationMs } = useRecordingStore.getState();
if (!reviewBlob) return;
@@ -55,10 +58,11 @@ export function useRecorder(streamId: string | null) {
const mimeType = reviewBlob.type || VIDEO_FALLBACK_MIME;
const fileName = `recording-${Date.now()}.webm`;
const { object, upload_url } = await apiClient.prepareUpload({
file_name: fileName,
const { object_id, upload_url } = await apiClient.prepareUpload({
network_id: networkId,
name: fileName,
content_type: mimeType,
size_bytes: reviewBlob.size,
content_length: reviewBlob.size,
});
await fetch(upload_url, {
@@ -67,12 +71,12 @@ export function useRecorder(streamId: string | null) {
body: reviewBlob,
});
await apiClient.confirmUpload(object.id);
await apiClient.confirmUpload(object_id);
const particle = await apiClient.createStreamParticle(streamId, {
type: "media",
data: {
object_id: object.id,
object_id,
duration_ms: reviewDurationMs,
mime_type: mimeType,
},
@@ -91,7 +95,7 @@ export function useRecorder(streamId: string | null) {
} catch (err) {
setError(err instanceof Error ? err.message : "Upload failed");
}
}, [streamId, setStatus, setError, resetRecording, addParticleToStream]);
}, [streamId, networkId, setStatus, setError, resetRecording, addParticleToStream]);
const startRecording = useCallback(async () => {
const currentStatus = useRecordingStore.getState().status;
js/src/pages/stream-player-page.tsx
+7 -23
View File
@@ -21,7 +21,6 @@ export function StreamPlayerPage() {
const particles = usePlaybackStore((s) => s.particles);
const currentIndex = usePlaybackStore((s) => s.currentIndex);
const status = usePlaybackStore((s) => s.status);
const initStream = usePlaybackStore((s) => s.initStream);
const next = usePlaybackStore((s) => s.next);
const prev = usePlaybackStore((s) => s.prev);
@@ -30,13 +29,14 @@ export function StreamPlayerPage() {
const pause = usePlaybackStore((s) => s.pause);
const resume = usePlaybackStore((s) => s.resume);
const { startRecording, stopRecording, cancelRecording, confirmSend } =
useRecorder(streamId ?? null);
// Find the stream and its parent network
const networkWithStream = networks.find((n) =>
n.streams.some((s) => s.id === streamId),
);
const stream = networkWithStream?.streams.find((s) => s.id === streamId);
// Find the stream across all networks
const stream = networks
.flatMap((n) => n.streams)
.find((s) => s.id === streamId);
const { startRecording, stopRecording, cancelRecording, confirmSend } =
useRecorder(streamId ?? null, networkWithStream?.id ?? null);
useEffect(() => {
if (!stream) return;
@@ -246,22 +246,6 @@ export function StreamPlayerPage() {
</div>
</div>
{/* End of stream overlay */}
{status === "ended" && (
<div className="absolute inset-0 z-20 flex items-center justify-center bg-black/80">
<div className="flex flex-col items-center gap-4">
<p className="text-sm text-white">End of stream</p>
<Button
variant="outline"
size="sm"
onClick={() => navigate("/")}
>
Back to streams
</Button>
</div>
</div>
)}
{/* Text compose overlay */}
{composingText && streamId && (
<TextComposeOverlay