From b77ae21973b6ee498cb9fcc8378a6a3112b2bed8 Mon Sep 17 00:00:00 2001 From: talksik Date: Mon, 3 Jan 2022 18:24:00 -0800 Subject: [PATCH 1/5] getting it sort of working but have a freeze --- nirvana-ios/Services/AuthSessionStore.swift | 6 +- .../Views/InnerCircle/CircleGridView.swift | 13 +++- .../InnerCircle/InnerCircleViewModel.swift | 78 ++++++++++++++----- 3 files changed, 71 insertions(+), 26 deletions(-) diff --git a/nirvana-ios/Services/AuthSessionStore.swift b/nirvana-ios/Services/AuthSessionStore.swift index 8a5b2ee..c6de9c3 100644 --- a/nirvana-ios/Services/AuthSessionStore.swift +++ b/nirvana-ios/Services/AuthSessionStore.swift @@ -33,7 +33,7 @@ final class AuthSessionStore: ObservableObject, SessionStore { @Published var sessionState: SessionState = SessionState.notCheckedYet // TODO: figure out which ones to publish - var messagesArr: [Message] = [] + @Published var messagesArr: [Message] = [] var userFriendsDict: [String: UserFriends] = [:] // all active and inactive relationships // transformed data for the views @@ -337,7 +337,7 @@ extension AuthSessionStore { // if this user is not already a friend, active or inactive/rejected, then get their user data and add to inbox if self.userFriendsDict.keys.contains(userFriend!.userId) { - print("already have this user in my circle or I rejected them \(userFriend?.userId)") +// print("already have this user in my circle or I rejected them \(userFriend?.userId)") continue } else { @@ -383,7 +383,7 @@ extension AuthSessionStore { self.messagesArr = documents.compactMap { (queryDocumentSnapshot) -> Message? in do { let currMessage = try queryDocumentSnapshot.data(as: Message.self) - print("new message received! \(currMessage!.sentTimestamp)") +// print("new message received! \(currMessage!.sentTimestamp)") DispatchQueue.main.async { if currMessage != nil { // not really possible but just check diff --git a/nirvana-ios/Views/InnerCircle/CircleGridView.swift b/nirvana-ios/Views/InnerCircle/CircleGridView.swift index 2fd6761..845d72a 100644 --- a/nirvana-ios/Views/InnerCircle/CircleGridView.swift +++ b/nirvana-ios/Views/InnerCircle/CircleGridView.swift @@ -368,10 +368,15 @@ struct CircleGridView: View { } self.animateLiveConvos = true - }// univseral alert TODO: move to inner circle view and use this pattern for all view models -// .alert(item: self.$convoVM.error) {error in -// error.alert -// } + } + .onReceive(self.authSessionStore.$messagesArr) {_ in + // TODO: don't know if this fires if I am active and I receive a message but that's not something I am allowing right now with the continuous convo feature + + print("new messages found!!!! on receive going to cache all messages to load them faster for you!") + + // TODO: might be including messages of inbox users? overloading memory? + self.innerCircleVM.cacheIncomingMessages(friendMessagesDict: self.authSessionStore.relevantMessagesByUserDict) + } } // scrollview reader } diff --git a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift index 7c75871..9f028ce 100644 --- a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift +++ b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift @@ -71,7 +71,9 @@ class InnerCircleViewModel: NSObject, ObservableObject { } var audioRecorder : AVAudioRecorder! + var queuePlayer = AVQueuePlayer() + private var cachedPlayerItemsDict: [String: Data] = [:] // url to player item @Published var isRecording : Bool = false @@ -255,15 +257,22 @@ extension InnerCircleViewModel { // clearing the player to make room for this friend's convo or to deselect this user self.stopPlayingAnyAudio() - var AVPlayerItems: [AVPlayerItem] = [] + // reset progress + self.messagesListeningProgress = Float(Self.multiplier * 2) + + var AVPlayerItems: [CachingPlayerItem] = [] for url in audioUrls { - let asset = AVAsset(url: url) - let playerItem = AVPlayerItem(asset: asset) - - // notification for when each playeritem is done playing -// NotificationCenter.default.addObserver(self, selector: #selector(self.playerDidFinishPlaying(sender:)), name: NSNotification.Name.AVPlayerItemDidPlayToEndTime, object: playerItem) - - AVPlayerItems.append(playerItem) + // if we have a player in the cache, then play it from there + if self.cachedPlayerItemsDict.keys.contains(url.absoluteString), let data = self.cachedPlayerItemsDict[url.absoluteString] { + // keep the cachedplayer item in tact as the player messes with those items + let cachedPlayerItem = CachingPlayerItem(data: data, mimeType: "audio/mp4", fileExtension: "m4a") + AVPlayerItems.append(cachedPlayerItem) + print("playing a message from cache") + } else { + let asset = AVAsset(url: url) + let playerItem = CachingPlayerItem(url: url) + AVPlayerItems.append(playerItem) + } } if AVPlayerItems.count <= 0 { @@ -271,6 +280,8 @@ extension InnerCircleViewModel { return } + print("current state of cached audio files: \(self.cachedPlayerItemsDict)") + // start playing if there are messages to listen to print("have \(AVPlayerItems.count) messages to play") @@ -284,9 +295,6 @@ extension InnerCircleViewModel { print("player queued up items!!!") - // reset progress - self.messagesListeningProgress = Float(Self.multiplier * 2) - DispatchQueue.global(qos: .background).async { self.addBoundaryTimeObserver(playerItems: AVPlayerItems) } @@ -357,14 +365,46 @@ extension InnerCircleViewModel { // self.timeObserverToken = nil // } // } + + func cacheIncomingMessages(friendMessagesDict: [String: [Message]]) { + // need way of making this get called whenever there are messages that come in but do all this in the background...onreceive of new messages, check if we already have it cached, and if not, then download + guard let userId = AuthSessionStore.getCurrentUserId() else {return} + DispatchQueue.global(qos: .background).async { + // get all relevant messages that the user may listen to + // don't want to cache all messages + for friendId in friendMessagesDict.keys { + let messagesRelatedToFriend = friendMessagesDict[friendId] ?? [] + + if messagesRelatedToFriend.count == 0 { + return + } + + for message in messagesRelatedToFriend { + // if it's starting to get to my messages then don't play + if message.senderId == userId { + break + } + + // only add to queue if we can convert the database url to a valid url here + if let audioUrl = URL(string: message.audioDataUrl) { // check if it's a valid url + if !self.cachedPlayerItemsDict.keys.contains(message.audioDataUrl) { + // save to play from mem later + let task = URLSession.shared.dataTask(with: audioUrl) {[weak self] (data, response, error) in + guard let data = data else { return } + print(data) + self?.cachedPlayerItemsDict[audioUrl.absoluteString] = data as? Data + } + + task.resume() + } + } + } + } + } + + // TODO: need way of deleting cache when we close app or something...can rile up the cost + } } -extension InnerCircleViewModel { -// @objc func playerDidFinishPlaying(sender: Notification) { -// // Your code here -// print("finished playing an item") -// -// self.numberofMessagesToPlay -= 1 -// } -} + From 98c551bc3336da0c754cd60199b42bf8883b78cc Mon Sep 17 00:00:00 2001 From: talksik Date: Mon, 3 Jan 2022 19:05:50 -0800 Subject: [PATCH 2/5] decent working method of storing data locally --- .../InnerCircle/InnerCircleViewModel.swift | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift index 9f028ce..7b20c08 100644 --- a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift +++ b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift @@ -73,7 +73,8 @@ class InnerCircleViewModel: NSObject, ObservableObject { var audioRecorder : AVAudioRecorder! var queuePlayer = AVQueuePlayer() - private var cachedPlayerItemsDict: [String: Data] = [:] // url to player item + private var cachedPlayerItemsDict: [String: URL] = [:] // firebase audio url to local url + private let documentDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] @Published var isRecording : Bool = false @@ -260,17 +261,18 @@ extension InnerCircleViewModel { // reset progress self.messagesListeningProgress = Float(Self.multiplier * 2) - var AVPlayerItems: [CachingPlayerItem] = [] + var AVPlayerItems: [AVPlayerItem] = [] for url in audioUrls { // if we have a player in the cache, then play it from there - if self.cachedPlayerItemsDict.keys.contains(url.absoluteString), let data = self.cachedPlayerItemsDict[url.absoluteString] { + if self.cachedPlayerItemsDict.keys.contains(url.absoluteString), let localUrl = self.cachedPlayerItemsDict[url.absoluteString] { // keep the cachedplayer item in tact as the player messes with those items - let cachedPlayerItem = CachingPlayerItem(data: data, mimeType: "audio/mp4", fileExtension: "m4a") - AVPlayerItems.append(cachedPlayerItem) + let asset = AVAsset(url: localUrl) + let playerItem = AVPlayerItem(asset: asset) + AVPlayerItems.append(playerItem) print("playing a message from cache") } else { let asset = AVAsset(url: url) - let playerItem = CachingPlayerItem(url: url) + let playerItem = AVPlayerItem(asset: asset) AVPlayerItems.append(playerItem) } } @@ -393,7 +395,13 @@ extension InnerCircleViewModel { let task = URLSession.shared.dataTask(with: audioUrl) {[weak self] (data, response, error) in guard let data = data else { return } print(data) - self?.cachedPlayerItemsDict[audioUrl.absoluteString] = data as? Data + + if let cacheFilePath = self?.getTemporaryDirectory().appendingPathComponent("\(UUID().uuidString).m4a") { + try? data.write(to: cacheFilePath) + + // save in local cache + self?.cachedPlayerItemsDict[audioUrl.absoluteString] = cacheFilePath + } } task.resume() From 4f4e5ba397207779bf654efce8cc40560b3e26f3 Mon Sep 17 00:00:00 2001 From: talksik Date: Mon, 3 Jan 2022 19:29:59 -0800 Subject: [PATCH 3/5] decent place but still buggy --- nirvana-ios/Services/AuthSessionStore.swift | 59 ++++++++++--------- .../InnerCircle/InnerCircleViewModel.swift | 9 ++- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/nirvana-ios/Services/AuthSessionStore.swift b/nirvana-ios/Services/AuthSessionStore.swift index c6de9c3..1eb2b23 100644 --- a/nirvana-ios/Services/AuthSessionStore.swift +++ b/nirvana-ios/Services/AuthSessionStore.swift @@ -380,42 +380,43 @@ extension AuthSessionStore { //optimize this? but also saving on memory and same db reads self.relevantMessagesByUserDict.removeAll() - self.messagesArr = documents.compactMap { (queryDocumentSnapshot) -> Message? in - do { - let currMessage = try queryDocumentSnapshot.data(as: Message.self) -// print("new message received! \(currMessage!.sentTimestamp)") - - DispatchQueue.main.async { - if currMessage != nil { // not really possible but just check - // if the user doesn't exist for the dictionary, then add it - // this means it's most likely someone new (never had user_friend relationship before) messaging for the user's inbox - // TODO: prolly want to make a call to get this sender user details for the inbox, but they should either be in the friendsDict or their are not a friend so won't be there - // also add in any messages where I am the sender - if currMessage!.senderId == currUserId { // if I am the sender - if self.relevantMessagesByUserDict[currMessage!.receiverId] == nil { - self.relevantMessagesByUserDict[currMessage!.receiverId] = [currMessage!] - } else { - self.relevantMessagesByUserDict[currMessage!.receiverId]?.append(currMessage!) + DispatchQueue.main.async { + self.messagesArr = documents.compactMap { (queryDocumentSnapshot) -> Message? in + do { + let currMessage = try queryDocumentSnapshot.data(as: Message.self) + // print("new message received! \(currMessage!.sentTimestamp)") + + DispatchQueue.main.async { + if currMessage != nil { // not really possible but just check + // if the user doesn't exist for the dictionary, then add it + // this means it's most likely someone new (never had user_friend relationship before) messaging for the user's inbox + // TODO: prolly want to make a call to get this sender user details for the inbox, but they should either be in the friendsDict or their are not a friend so won't be there + // also add in any messages where I am the sender + if currMessage!.senderId == currUserId { // if I am the sender + if self.relevantMessagesByUserDict[currMessage!.receiverId] == nil { + self.relevantMessagesByUserDict[currMessage!.receiverId] = [currMessage!] + } else { + self.relevantMessagesByUserDict[currMessage!.receiverId]?.append(currMessage!) + } } - } - else { // if I am receiving - if self.relevantMessagesByUserDict[currMessage!.senderId] == nil { - self.relevantMessagesByUserDict[currMessage!.senderId] = [currMessage!] - } else { - self.relevantMessagesByUserDict[currMessage!.senderId]?.append(currMessage!) + else { // if I am receiving + if self.relevantMessagesByUserDict[currMessage!.senderId] == nil { + self.relevantMessagesByUserDict[currMessage!.senderId] = [currMessage!] + } else { + self.relevantMessagesByUserDict[currMessage!.senderId]?.append(currMessage!) + } } } } - self.objectWillChange.send() + return currMessage + } catch { + print(error) } - - return currMessage - } catch { - print(error) + return nil } - return nil - } + } + } self.listenersActive = true diff --git a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift index 7b20c08..746f69e 100644 --- a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift +++ b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift @@ -79,7 +79,7 @@ class InnerCircleViewModel: NSObject, ObservableObject { @Published var isRecording : Bool = false @Published var messagesListeningProgress: Float = 1.0 - static let multiplier: Float64 = 0.05 + static let multiplier: Float64 = 0.01 let audioSession = AVAudioSession.sharedInstance() @@ -259,7 +259,7 @@ extension InnerCircleViewModel { self.stopPlayingAnyAudio() // reset progress - self.messagesListeningProgress = Float(Self.multiplier * 2) + self.messagesListeningProgress = Float(0) var AVPlayerItems: [AVPlayerItem] = [] for url in audioUrls { @@ -319,8 +319,6 @@ extension InnerCircleViewModel { currentTime = currentTime + interval times.append(NSValue(time: currentTime)) } - // this last one to make sure we get a full loop - times.append(NSValue(time: totalDuration)) // Add time observer. Observe boundary time changes on the main queue. // TODO: not hitting the last one/100% @@ -369,7 +367,7 @@ extension InnerCircleViewModel { // } func cacheIncomingMessages(friendMessagesDict: [String: [Message]]) { - // need way of making this get called whenever there are messages that come in but do all this in the background...onreceive of new messages, check if we already have it cached, and if not, then download + // TODO: not firing for some reason onreceive of new data or not including new data guard let userId = AuthSessionStore.getCurrentUserId() else {return} DispatchQueue.global(qos: .background).async { @@ -392,6 +390,7 @@ extension InnerCircleViewModel { if let audioUrl = URL(string: message.audioDataUrl) { // check if it's a valid url if !self.cachedPlayerItemsDict.keys.contains(message.audioDataUrl) { // save to play from mem later + let task = URLSession.shared.dataTask(with: audioUrl) {[weak self] (data, response, error) in guard let data = data else { return } print(data) From e1735bb8ed34ddb625d4ca5abcb48e0cb508ddda Mon Sep 17 00:00:00 2001 From: talksik Date: Mon, 3 Jan 2022 19:45:35 -0800 Subject: [PATCH 4/5] good enough fix for the problems --- nirvana-ios/Services/AuthSessionStore.swift | 34 +++++++++---------- .../InnerCircle/InnerCircleViewModel.swift | 10 ++++-- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/nirvana-ios/Services/AuthSessionStore.swift b/nirvana-ios/Services/AuthSessionStore.swift index 1eb2b23..f0f25b2 100644 --- a/nirvana-ios/Services/AuthSessionStore.swift +++ b/nirvana-ios/Services/AuthSessionStore.swift @@ -386,25 +386,23 @@ extension AuthSessionStore { let currMessage = try queryDocumentSnapshot.data(as: Message.self) // print("new message received! \(currMessage!.sentTimestamp)") - DispatchQueue.main.async { - if currMessage != nil { // not really possible but just check - // if the user doesn't exist for the dictionary, then add it - // this means it's most likely someone new (never had user_friend relationship before) messaging for the user's inbox - // TODO: prolly want to make a call to get this sender user details for the inbox, but they should either be in the friendsDict or their are not a friend so won't be there - // also add in any messages where I am the sender - if currMessage!.senderId == currUserId { // if I am the sender - if self.relevantMessagesByUserDict[currMessage!.receiverId] == nil { - self.relevantMessagesByUserDict[currMessage!.receiverId] = [currMessage!] - } else { - self.relevantMessagesByUserDict[currMessage!.receiverId]?.append(currMessage!) - } + if currMessage != nil { // not really possible but just check + // if the user doesn't exist for the dictionary, then add it + // this means it's most likely someone new (never had user_friend relationship before) messaging for the user's inbox + // TODO: prolly want to make a call to get this sender user details for the inbox, but they should either be in the friendsDict or their are not a friend so won't be there + // also add in any messages where I am the sender + if currMessage!.senderId == currUserId { // if I am the sender + if self.relevantMessagesByUserDict[currMessage!.receiverId] == nil { + self.relevantMessagesByUserDict[currMessage!.receiverId] = [currMessage!] + } else { + self.relevantMessagesByUserDict[currMessage!.receiverId]?.append(currMessage!) } - else { // if I am receiving - if self.relevantMessagesByUserDict[currMessage!.senderId] == nil { - self.relevantMessagesByUserDict[currMessage!.senderId] = [currMessage!] - } else { - self.relevantMessagesByUserDict[currMessage!.senderId]?.append(currMessage!) - } + } + else { // if I am receiving + if self.relevantMessagesByUserDict[currMessage!.senderId] == nil { + self.relevantMessagesByUserDict[currMessage!.senderId] = [currMessage!] + } else { + self.relevantMessagesByUserDict[currMessage!.senderId]?.append(currMessage!) } } } diff --git a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift index 746f69e..7881a3f 100644 --- a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift +++ b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift @@ -282,8 +282,6 @@ extension InnerCircleViewModel { return } - print("current state of cached audio files: \(self.cachedPlayerItemsDict)") - // start playing if there are messages to listen to print("have \(AVPlayerItems.count) messages to play") @@ -308,6 +306,11 @@ extension InnerCircleViewModel { totalDuration = CMTimeAdd(totalDuration, item.asset.duration) } + // TODO: not hitting the last one/100% + // it either plays on time or there is a little lag on multiple items... + // offset this by adding 1 when it reaches the totalDuration as it's longer + totalDuration = CMTimeSubtract(totalDuration, CMTimeMakeWithSeconds(1, preferredTimescale: 1)) + var times = [NSValue]() // Set initial time to zero var currentTime = CMTime.zero @@ -320,8 +323,9 @@ extension InnerCircleViewModel { times.append(NSValue(time: currentTime)) } + + // Add time observer. Observe boundary time changes on the main queue. - // TODO: not hitting the last one/100% self.queuePlayer.addBoundaryTimeObserver(forTimes: times, queue: .main) { [weak self] in // Update UI self?.messagesListeningProgress += Float(Self.multiplier) From e13ad4b928be0bbca05a4c36e9d157eb723402bb Mon Sep 17 00:00:00 2001 From: talksik Date: Mon, 3 Jan 2022 19:46:43 -0800 Subject: [PATCH 5/5] Update InnerCircleViewModel.swift --- nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift index 7881a3f..14d4591 100644 --- a/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift +++ b/nirvana-ios/Views/InnerCircle/InnerCircleViewModel.swift @@ -74,7 +74,6 @@ class InnerCircleViewModel: NSObject, ObservableObject { var queuePlayer = AVQueuePlayer() private var cachedPlayerItemsDict: [String: URL] = [:] // firebase audio url to local url - private let documentDirectory = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] @Published var isRecording : Bool = false