different paradigm of managing audio

.vscode/settings.json

@@ -8,7 +8,7 @@
   "editor.defaultFormatter": "esbenp.prettier-vscode",
   "workbench.colorTheme": "Default Light+",
   "javascript.format.semicolons": "insert",
-  "window.zoomLevel": 1
+  "window.zoomLevel": 0
 
   // "eslint.workingDirectories": ["./packages/desktop"]
 }

packages/desktop/src/providers/ConversationProvider.tsx

@@ -63,12 +63,18 @@ function useSocketFire() {
   return { handleConnectToLine, handleTuneIntoLine, handleUntuneFromLine };
 }
 
-const videoConstraints = {
+const VIDEO_CONSTRAINTS = {
   frameRate: 15,
   width: { max: 720, ideal: 720, min: 100 },
   height: { max: 720, ideal: 720, min: 100 },
 };
 
+const AUDIO_CONSTRAINTS = {
+  channelCount: {
+    ideal: 2,
+  },
+};
+
 interface IConversationContext {
   conversationMap: ConversationMap;
 
@@ -86,8 +92,7 @@ interface IConversationContext {
   updateConversationPriority?: (conversationId: string, newState: MemberState) => Promise<void>;
 
   userLocalStream?: MediaStream;
-  handleCastVideo?: () => void;
+  handleToggleVideo?: () => void;
-  handleStopVideo?: () => void;
 }
 
 const ConversationContext = React.createContext<IConversationContext>({
@@ -298,6 +303,9 @@ export function ConversationProvider({ children }: { children: React.ReactNode }
 
         return conversationToSelect;
       });
+
+      // !!TODO: go through each peer connection in the selected room and add the current stream
+      // remove/stop casting stream to previously selected room
     },
     [
       user,
@@ -447,27 +455,57 @@ export function ConversationProvider({ children }: { children: React.ReactNode }
   );
 
   const [userLocalStream, setUserLocalStream] = useState<MediaStream>();
+  const [userVideo, setUserVideo] = useState<boolean>(true);
+  const [userAudio, setUserAudio] = useState<boolean>(true);
 
-  const handleCastVideo = useCallback(() => {
+  useEffect(() => {
-    // create a new stream
+    // want to grab device stream even if not selected a room
-    // if there's a selected conversation, then send it to that conversation
+    // only upload this stream to a room if we have a selected room
-    navigator.mediaDevices
-      .getUserMedia({ video: videoConstraints, audio: false })
-      .then((userStream: MediaStream) => {
-        setUserLocalStream(userStream);
-      })
-      .catch((error) => toast.error('problem getting video'));
-  }, [setUserLocalStream]);
 
-  const handleStopVideo = useCallback(() => {
+    if (userVideo || userAudio) {
-    setUserLocalStream((prevStream) => {
+      navigator.mediaDevices
-      if (prevStream) {
+        .getUserMedia({
-        prevStream.getTracks().map((track) => track.stop());
+          audio: userAudio ? AUDIO_CONSTRAINTS : false,
-      }
+          video: userVideo ? VIDEO_CONSTRAINTS : false,
+        })
+        .then((userStream) => {
+          console.log(`got new user stream`, userStream);
 
-      return undefined;
+          // set global stream for slight feedback while talking
-    });
+          setUserLocalStream(userStream);
-  }, [setUserLocalStream]);
+        });
+    } else {
+      setUserLocalStream(undefined);
+      // stop streaming to room now
+    }
+  }, [userVideo, userAudio, setUserLocalStream]);
+
+  useEffect(() => {
+    if (!userAudio) {
+      userLocalStream.getAudioTracks().map((track) => track.stop());
+    }
+  }, [userAudio, userLocalStream]);
+  useEffect(() => {
+    if (!userVideo) {
+      userLocalStream.getVideoTracks().map((track) => track.stop());
+    }
+  }, [userVideo, userLocalStream]);
+
+  // take the local stream and use simple peer add track
+  const handleToggleAudio = useCallback(() => {
+    setUserAudio((prevVal) => !prevVal);
+  }, []);
+
+  const handleToggleVideo = useCallback(() => {
+    setUserVideo((prevVal) => !prevVal);
+  }, [setUserVideo]);
+
+  // TODO: replace track of the same stream and
+  const handleShareScreen = useCallback(() => {
+    //
+  }, []);
+
+  useKeyPressEvent(KeyboardShortcuts.speak.shortcutKey, handleToggleAudio, handleToggleAudio);
 
   const handleEscape = useCallback(() => {
     selectConversation(undefined);
@@ -508,8 +546,7 @@ export function ConversationProvider({ children }: { children: React.ReactNode }
         handleEscape,
         updateConversationPriority: updateConversationPriorityHandler,
         userLocalStream,
-        handleStopVideo,
+        handleToggleVideo,
-        handleCastVideo,
       }}
     >
       {children}

packages/desktop/src/tree/FooterControls.tsx

@@ -46,13 +46,8 @@ import useZen from '../providers/ZenProvider';
 export default function FooterControls() {
   const { handleFlowState } = useZen();
 
-  const {
+  const { selectedConversation, priorityConversations, userLocalStream, handleToggleVideo } =
-    selectedConversation,
+    useConversations();
-    priorityConversations,
-    userLocalStream,
-    handleCastVideo,
-    handleStopVideo,
-  } = useConversations();
   const { user, handleLogout } = useAuth();
 
   const { handleToggleDesktopMode, desktopMode } = useElectron();
@@ -82,17 +77,17 @@ export default function FooterControls() {
 
   const [isVideoOn, setIsVideoOn] = useState<boolean>(false);
 
-  const handleToggleVideo = useCallback(() => {
+  // const handleToggleVideo = useCallback(() => {
-    setIsVideoOn((prevVal) => {
+  //   setIsVideoOn((prevVal) => {
-      if (!prevVal) {
+  //     if (!prevVal) {
-        handleCastVideo();
+  //       handleCastVideo();
-      } else {
+  //     } else {
-        handleStopVideo();
+  //       handleStopVideo();
-      }
+  //     }
 
-      return !prevVal;
+  //     return !prevVal;
-    });
+  //   });
-  }, [setIsVideoOn, handleCastVideo, handleStopVideo]);
+  // }, [setIsVideoOn, handleCastVideo, handleStopVideo]);
 
   const handleCloseUserSettings = useCallback(() => {
     setOpenUserSettings(false);
@@ -183,7 +178,7 @@ export default function FooterControls() {
           aria-haspopup="true"
           aria-expanded={open ? 'true' : undefined}
         >
-          {isVideoOn ? (
+          {userLocalStream ? (
             <Box height={60} width={40} ref={localVideoRef} component={'video'} autoPlay />
           ) : (
             <Avatar alt={user.givenName} src={user.picture} />
@@ -243,7 +238,9 @@ function OverlayConversation({
 
   const userVideoRef = useRef<HTMLVideoElement>(null);
 
-  useEffect(() => {}, []);
+  useEffect(() => {
+    //
+  }, []);
 
   return (
     <Stack

packages/desktop/src/tree/MainPanel.tsx

@@ -308,7 +308,7 @@ function ConversationDetails({ masterConversation }: { masterConversation: Maste
       <Box sx={{ p: 2, boxShadow: 3, zIndex: 5 }} flexDirection={'column'} mt={'auto'}>
         {/* live video and screenshares */}
         <Stack direction="row" alignItems={'center'}>
-          {masterConversation.room &&
+          {/* {masterConversation.room &&
             Object.entries(masterConversation.room).map(([userId, userPeerContents]) => {
               return (
                 <Video
@@ -316,7 +316,7 @@ function ConversationDetails({ masterConversation }: { masterConversation: Maste
                   stream={userPeerContents.stream}
                 />
               );
-            })}
+            })} */}
         </Stack>
 
         <Stack direction="row" alignItems="center" justifyContent={'flex-end'} gap={1}>

packages/desktop/src/util/keyboard.ts

@@ -9,4 +9,8 @@ export const KeyboardShortcuts: Record<string, Shortcut> = {
     shortcutKey: 'Escape',
     label: 'esc',
   },
+  speak: {
+    shortcutKey: '`',
+    label: '~',
+  },
 };

packages/desktop/src/util/types.ts

@@ -15,15 +15,15 @@ export type MasterConversation = Conversation & {
   room?: {
     [userId: string]: {
       peer: Peer;
+
+      //the remote stream of this peer
       stream?: MediaStream;
-      audioTrack?: MediaStreamTrack;
-      videoTrack?: MediaStreamTrack;
-      screenTrack?: MediaStreamTrack;
 
       // calling happening whether it is a newbie or master
       isConnecting?: boolean;
     };
   };
+  localStream?: MediaStream;
 
   // all of audio clips, links, media, etc.
   content?: ContentBlock[];