From e1f03e0efaceb82277e7c9873aefdf01b8ef7cb5 Mon Sep 17 00:00:00 2001 From: talksik Date: Wed, 29 Jun 2022 10:12:03 -0500 Subject: [PATCH] different paradigm of managing audio --- .vscode/settings.json | 2 +- .../src/providers/ConversationProvider.tsx | 83 ++++++++++++++----- packages/desktop/src/tree/FooterControls.tsx | 35 ++++---- packages/desktop/src/tree/MainPanel.tsx | 4 +- packages/desktop/src/util/keyboard.ts | 4 + packages/desktop/src/util/types.ts | 6 +- 6 files changed, 86 insertions(+), 48 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index abfc7e4..23131b1 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,7 +8,7 @@ "editor.defaultFormatter": "esbenp.prettier-vscode", "workbench.colorTheme": "Default Light+", "javascript.format.semicolons": "insert", - "window.zoomLevel": 1 + "window.zoomLevel": 0 // "eslint.workingDirectories": ["./packages/desktop"] } diff --git a/packages/desktop/src/providers/ConversationProvider.tsx b/packages/desktop/src/providers/ConversationProvider.tsx index a33aa2c..ad389a8 100644 --- a/packages/desktop/src/providers/ConversationProvider.tsx +++ b/packages/desktop/src/providers/ConversationProvider.tsx @@ -63,12 +63,18 @@ function useSocketFire() { return { handleConnectToLine, handleTuneIntoLine, handleUntuneFromLine }; } -const videoConstraints = { +const VIDEO_CONSTRAINTS = { frameRate: 15, width: { max: 720, ideal: 720, min: 100 }, height: { max: 720, ideal: 720, min: 100 }, }; +const AUDIO_CONSTRAINTS = { + channelCount: { + ideal: 2, + }, +}; + interface IConversationContext { conversationMap: ConversationMap; @@ -86,8 +92,7 @@ interface IConversationContext { updateConversationPriority?: (conversationId: string, newState: MemberState) => Promise; userLocalStream?: MediaStream; - handleCastVideo?: () => void; - handleStopVideo?: () => void; + handleToggleVideo?: () => void; } const ConversationContext = React.createContext({ @@ -298,6 +303,9 @@ export function ConversationProvider({ children }: { children: React.ReactNode } return conversationToSelect; }); + + // !!TODO: go through each peer connection in the selected room and add the current stream + // remove/stop casting stream to previously selected room }, [ user, @@ -447,27 +455,57 @@ export function ConversationProvider({ children }: { children: React.ReactNode } ); const [userLocalStream, setUserLocalStream] = useState(); + const [userVideo, setUserVideo] = useState(true); + const [userAudio, setUserAudio] = useState(true); - const handleCastVideo = useCallback(() => { - // create a new stream - // if there's a selected conversation, then send it to that conversation - navigator.mediaDevices - .getUserMedia({ video: videoConstraints, audio: false }) - .then((userStream: MediaStream) => { - setUserLocalStream(userStream); - }) - .catch((error) => toast.error('problem getting video')); - }, [setUserLocalStream]); + useEffect(() => { + // want to grab device stream even if not selected a room + // only upload this stream to a room if we have a selected room - const handleStopVideo = useCallback(() => { - setUserLocalStream((prevStream) => { - if (prevStream) { - prevStream.getTracks().map((track) => track.stop()); - } + if (userVideo || userAudio) { + navigator.mediaDevices + .getUserMedia({ + audio: userAudio ? AUDIO_CONSTRAINTS : false, + video: userVideo ? VIDEO_CONSTRAINTS : false, + }) + .then((userStream) => { + console.log(`got new user stream`, userStream); - return undefined; - }); - }, [setUserLocalStream]); + // set global stream for slight feedback while talking + setUserLocalStream(userStream); + }); + } else { + setUserLocalStream(undefined); + // stop streaming to room now + } + }, [userVideo, userAudio, setUserLocalStream]); + + useEffect(() => { + if (!userAudio) { + userLocalStream.getAudioTracks().map((track) => track.stop()); + } + }, [userAudio, userLocalStream]); + useEffect(() => { + if (!userVideo) { + userLocalStream.getVideoTracks().map((track) => track.stop()); + } + }, [userVideo, userLocalStream]); + + // take the local stream and use simple peer add track + const handleToggleAudio = useCallback(() => { + setUserAudio((prevVal) => !prevVal); + }, []); + + const handleToggleVideo = useCallback(() => { + setUserVideo((prevVal) => !prevVal); + }, [setUserVideo]); + + // TODO: replace track of the same stream and + const handleShareScreen = useCallback(() => { + // + }, []); + + useKeyPressEvent(KeyboardShortcuts.speak.shortcutKey, handleToggleAudio, handleToggleAudio); const handleEscape = useCallback(() => { selectConversation(undefined); @@ -508,8 +546,7 @@ export function ConversationProvider({ children }: { children: React.ReactNode } handleEscape, updateConversationPriority: updateConversationPriorityHandler, userLocalStream, - handleStopVideo, - handleCastVideo, + handleToggleVideo, }} > {children} diff --git a/packages/desktop/src/tree/FooterControls.tsx b/packages/desktop/src/tree/FooterControls.tsx index 42a63a7..b54f683 100644 --- a/packages/desktop/src/tree/FooterControls.tsx +++ b/packages/desktop/src/tree/FooterControls.tsx @@ -46,13 +46,8 @@ import useZen from '../providers/ZenProvider'; export default function FooterControls() { const { handleFlowState } = useZen(); - const { - selectedConversation, - priorityConversations, - userLocalStream, - handleCastVideo, - handleStopVideo, - } = useConversations(); + const { selectedConversation, priorityConversations, userLocalStream, handleToggleVideo } = + useConversations(); const { user, handleLogout } = useAuth(); const { handleToggleDesktopMode, desktopMode } = useElectron(); @@ -82,17 +77,17 @@ export default function FooterControls() { const [isVideoOn, setIsVideoOn] = useState(false); - const handleToggleVideo = useCallback(() => { - setIsVideoOn((prevVal) => { - if (!prevVal) { - handleCastVideo(); - } else { - handleStopVideo(); - } + // const handleToggleVideo = useCallback(() => { + // setIsVideoOn((prevVal) => { + // if (!prevVal) { + // handleCastVideo(); + // } else { + // handleStopVideo(); + // } - return !prevVal; - }); - }, [setIsVideoOn, handleCastVideo, handleStopVideo]); + // return !prevVal; + // }); + // }, [setIsVideoOn, handleCastVideo, handleStopVideo]); const handleCloseUserSettings = useCallback(() => { setOpenUserSettings(false); @@ -183,7 +178,7 @@ export default function FooterControls() { aria-haspopup="true" aria-expanded={open ? 'true' : undefined} > - {isVideoOn ? ( + {userLocalStream ? ( ) : ( @@ -243,7 +238,9 @@ function OverlayConversation({ const userVideoRef = useRef(null); - useEffect(() => {}, []); + useEffect(() => { + // + }, []); return ( {/* live video and screenshares */} - {masterConversation.room && + {/* {masterConversation.room && Object.entries(masterConversation.room).map(([userId, userPeerContents]) => { return ( diff --git a/packages/desktop/src/util/keyboard.ts b/packages/desktop/src/util/keyboard.ts index e453595..776ec0f 100644 --- a/packages/desktop/src/util/keyboard.ts +++ b/packages/desktop/src/util/keyboard.ts @@ -9,4 +9,8 @@ export const KeyboardShortcuts: Record = { shortcutKey: 'Escape', label: 'esc', }, + speak: { + shortcutKey: '`', + label: '~', + }, }; diff --git a/packages/desktop/src/util/types.ts b/packages/desktop/src/util/types.ts index a6500ea..4544f24 100644 --- a/packages/desktop/src/util/types.ts +++ b/packages/desktop/src/util/types.ts @@ -15,15 +15,15 @@ export type MasterConversation = Conversation & { room?: { [userId: string]: { peer: Peer; + + //the remote stream of this peer stream?: MediaStream; - audioTrack?: MediaStreamTrack; - videoTrack?: MediaStreamTrack; - screenTrack?: MediaStreamTrack; // calling happening whether it is a newbie or master isConnecting?: boolean; }; }; + localStream?: MediaStream; // all of audio clips, links, media, etc. content?: ContentBlock[];