From f91fbfc79d6fe01f9d39a7d752a5d31a70d08b25 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Aug 2022 16:59:03 +0800 Subject: [PATCH 1/4] Return success when scrubbing via media controls --- .../OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift index 00030fb8a..b267d14cc 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift @@ -294,6 +294,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { commandCenter.changePlaybackPositionCommand.addTarget { event -> MPRemoteCommandHandlerStatus in if let event = event as? MPChangePlaybackPositionCommandEvent { self.player?.currentTime = event.positionTime + return .success } return .commandFailed } From 24901f9c0129703ef44237aff49f01ec813a3127 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Aug 2022 17:00:32 +0800 Subject: [PATCH 2/4] add shadow back to miniplayer --- .../OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift index 881989408..b413c200b 100644 --- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift +++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift @@ -246,7 +246,7 @@ public struct MiniPlayer: View { .padding(EdgeInsets(top: 0, leading: expanded ? 24 : 6, bottom: 0, trailing: expanded ? 24 : 6)) .background( Color.systemBackground - .shadow(color: expanded ? .clear : .gray /* .opacity(0.33) */, radius: 8, x: 0, y: 4) + .shadow(color: expanded ? .clear : .gray.opacity(0.33), radius: 8, x: 0, y: 4) .mask(Rectangle().padding(.top, -20)) ) .onTapGesture { From 886d5fbe2ccb761dde8f76fbe1f8a0ffe4b1a631 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Aug 2022 22:18:01 +0800 Subject: [PATCH 3/4] Add a share button for exporting MP3s, mostly useful for debugging --- .../App/Views/AudioPlayer/MiniPlayer.swift | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift index b413c200b..b66552bc9 100644 --- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift +++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift @@ -88,6 +88,26 @@ public struct MiniPlayer: View { .padding(.top, 8) .frame(maxWidth: .infinity, alignment: .leading) + Button( + action: { + let shareActivity = UIActivityViewController(activityItems: [self.audioSession.localAudioUrl], applicationActivities: nil) + if let vc = UIApplication.shared.windows.first?.rootViewController { + shareActivity.popoverPresentationController?.sourceView = vc.view + // Setup share activity position on screen on bottom center + shareActivity.popoverPresentationController?.sourceRect = CGRect(x: UIScreen.main.bounds.width / 2, y: UIScreen.main.bounds.height, width: 0, height: 0) + shareActivity.popoverPresentationController?.permittedArrowDirections = UIPopoverArrowDirection.down + vc.present(shareActivity, animated: true, completion: nil) + } + }, + label: { + Image(systemName: "square.and.arrow.up") + .font(.appCallout) + .tint(.appGrayText) + } + ) + .padding(.top, 8) + .frame(maxWidth: .infinity, alignment: .trailing) + Capsule() .fill(.gray) .frame(width: 60, height: 4) From 4837a63ecbb48ad0f7710f49b5427c58159da96f Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 24 Aug 2022 22:19:01 +0800 Subject: [PATCH 4/4] Preload audio files when the user opens the app This kicks off generation and download of audio files. Pending files are put in a queue and re-attempted. In the UI if a user hits the pending state they just get some UI now. --- .../Components/FeedCardNavigationLink.swift | 6 +- .../App/Views/Home/HomeFeedViewIOS.swift | 7 +- .../App/Views/Home/HomeFeedViewModel.swift | 7 +- .../Services/AudioSession/AudioSession.swift | 109 +++++++++++++----- 4 files changed, 94 insertions(+), 35 deletions(-) diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift b/apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift index 1f965db92..e43477839 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift @@ -5,6 +5,7 @@ import Views struct FeedCardNavigationLink: View { @EnvironmentObject var dataService: DataService + @EnvironmentObject var audioSession: AudioSession let item: LinkedItem @@ -33,7 +34,7 @@ struct FeedCardNavigationLink: View { .opacity(0) .buttonStyle(PlainButtonStyle()) .onAppear { - Task { await viewModel.itemAppeared(item: item, dataService: dataService) } + Task { await viewModel.itemAppeared(item: item, dataService: dataService, audioSession: audioSession) } } FeedCard(item: item) { viewModel.selectedLinkItem = item.objectID @@ -44,6 +45,7 @@ struct FeedCardNavigationLink: View { struct GridCardNavigationLink: View { @EnvironmentObject var dataService: DataService + @EnvironmentObject var audioSession: AudioSession @State private var scale = 1.0 @@ -86,7 +88,7 @@ struct GridCardNavigationLink: View { withAnimation { tapAction() } }) .onAppear { - Task { await viewModel.itemAppeared(item: item, dataService: dataService) } + Task { await viewModel.itemAppeared(item: item, dataService: dataService, audioSession: audioSession) } } } .aspectRatio(1.8, contentMode: .fill) diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift index bb53c910c..35eb891ab 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift @@ -11,11 +11,13 @@ import Views struct HomeFeedContainerView: View { @EnvironmentObject var dataService: DataService + @EnvironmentObject var audioSession: AudioSession + @AppStorage(UserDefaultKey.homeFeedlayoutPreference.rawValue) var prefersListLayout = false @ObservedObject var viewModel: HomeFeedViewModel func loadItems(isRefresh: Bool) { - Task { await viewModel.loadItems(dataService: dataService, isRefresh: isRefresh) } + Task { await viewModel.loadItems(dataService: dataService, audioSession: audioSession, isRefresh: isRefresh) } } var body: some View { @@ -339,6 +341,7 @@ import Views struct HomeFeedGridView: View { @EnvironmentObject var dataService: DataService + @EnvironmentObject var audioSession: AudioSession @State private var itemToRemove: LinkedItem? @State private var confirmationShown = false @@ -361,7 +364,7 @@ import Views } func loadItems(isRefresh: Bool) { - Task { await viewModel.loadItems(dataService: dataService, isRefresh: isRefresh) } + Task { await viewModel.loadItems(dataService: dataService, audioSession: audioSession, isRefresh: isRefresh) } } var body: some View { diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift index c4a6809cd..cafd507ea 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift @@ -40,14 +40,14 @@ import Views var searchIdx = 0 var receivedIdx = 0 - func itemAppeared(item: LinkedItem, dataService: DataService) async { + func itemAppeared(item: LinkedItem, dataService: DataService, audioSession: AudioSession) async { if isLoading { return } let itemIndex = items.firstIndex(where: { $0.id == item.id }) let thresholdIndex = items.index(items.endIndex, offsetBy: -5) // Check if user has scrolled to the last five items in the list if let itemIndex = itemIndex, itemIndex > thresholdIndex, items.count < thresholdIndex + 10 { - await loadItems(dataService: dataService, isRefresh: false) + await loadItems(dataService: dataService, audioSession: audioSession, isRefresh: false) } } @@ -55,7 +55,7 @@ import Views items.insert(item, at: 0) } - func loadItems(dataService: DataService, isRefresh: Bool) async { + func loadItems(dataService: DataService, audioSession: AudioSession, isRefresh: Bool) async { let syncStartTime = Date() let thisSearchIdx = searchIdx searchIdx += 1 @@ -123,6 +123,7 @@ import Views cursor = queryResult.cursor if let username = dataService.currentViewer?.username { await dataService.prefetchPages(itemIDs: newItems.map(\.unwrappedID), username: username) + await audioSession.preload(itemIDs: newItems.map(\.unwrappedID)) } } else { updateFetchController(dataService: dataService) diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift index b267d14cc..23f956066 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift @@ -66,6 +66,54 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { downloadTask?.cancel() } + public func preload(itemIDs: [String], retryCount: Int = 0) async { + var pendingList = [String]() + + for pageId in itemIDs { + let permFile = pathForAudioFile(pageId: pageId) + if FileManager.default.fileExists(atPath: permFile.path) { + print("audio file already downloaded: ", permFile) + continue + } + + // Attempt to fetch the file if not downloaded already + let result = try? await downloadAudioFile(pageId: pageId) + if result == nil { + print("audio file had error downloading: ", pageId) + pendingList.append(pageId) + } + + if let result = result, result.pending { + print("audio file is pending download: ", pageId) + pendingList.append(pageId) + } + } + + print("audio files pending download: ", pendingList) + if pendingList.isEmpty { + return + } + + if retryCount > 5 { + print("reached max preload depth, stopping preloading") + return + } + + let retryDelayInNanoSeconds = UInt64(retryCount * 2 * 1_000_000_000) + try? await Task.sleep(nanoseconds: retryDelayInNanoSeconds) + + await preload(itemIDs: pendingList, retryCount: retryCount + 1) + } + + public var localAudioUrl: URL? { + if let pageId = item?.id { + return FileManager.default + .urls(for: .documentDirectory, in: .userDomainMask)[0] + .appendingPathComponent(pageId + ".mp3") + } + return nil + } + public var scrubState: PlayerScrubState = .reset { didSet { switch scrubState { @@ -103,6 +151,16 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { } } + public func fileNameForAudioFile(_ pageId: String) -> String { + pageId + "-" + currentVoice + ".mp3" + } + + public func pathForAudioFile(pageId: String) -> URL { + FileManager.default + .urls(for: .documentDirectory, in: .userDomainMask)[0] + .appendingPathComponent(fileNameForAudioFile(pageId)) + } + public func startAudio() { state = .loading setupNotifications() @@ -110,19 +168,24 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { let pageId = item!.unwrappedID downloadTask = Task { - do { - _ = try await downloadAudioFile(pageId: pageId) - if Task.isCancelled { return } - DispatchQueue.main.async { - self.startDownloadedAudioFile(pageId: pageId) - } - } catch { - // TODO: display a failure toast here + let result = try? await downloadAudioFile(pageId: pageId) + if Task.isCancelled { return } + + if result == nil { DispatchQueue.main.async { + NSNotification.operationSuccess(message: "Error generating audio.") self.stop() } - print("FAILED TO DOWNLOAD AUDIO URL") - print(error) + } + + if let result = result, result.pending { + DispatchQueue.main.async { + NSNotification.operationSuccess(message: "Your audio is being generated.") + } + } + + DispatchQueue.main.async { + self.startDownloadedAudioFile(pageId: pageId) } } } @@ -136,10 +199,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { // TODO: Maybe check if app is active so it doesn't end up playing later? - let audioUrl = FileManager.default - .urls(for: .documentDirectory, in: .userDomainMask)[0] - .appendingPathComponent(pageId + ".mp3") - + let audioUrl = pathForAudioFile(pageId: pageId) if !FileManager.default.fileExists(atPath: audioUrl.path) { stop() return @@ -300,16 +360,12 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { } } - func downloadAudioFile(pageId: String) async throws -> URL? { - let audioUrl = FileManager.default - .urls(for: .documentDirectory, in: .userDomainMask)[0] - .appendingPathComponent(pageId + ".mp3") + func downloadAudioFile(pageId: String) async throws -> (pending: Bool, url: URL?) { + let audioUrl = pathForAudioFile(pageId: pageId) -// if FileManager.default.fileExists(atPath: audioUrl.path) { -// // Prevent re-download -// // TODO: We aren't doing this very safely, we should be verifying a checksum -// return audioUrl -// } + if FileManager.default.fileExists(atPath: audioUrl.path) { + return (pending: false, url: audioUrl) + } guard let url = URL(string: "/api/article/\(pageId)/mp3/\(currentVoice)", relativeTo: appEnvironment.serverBaseURL) else { throw BasicError.message(messageText: "Invalid audio URL") @@ -328,10 +384,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { } print("httpResponse: ", httpResponse) if let httpResponse = result?.1 as? HTTPURLResponse, httpResponse.statusCode == 202 { - print("Tell the user the download has been queued") - DispatchQueue.main.async { - NSNotification.operationSuccess(message: "Your audio is being created.") - } + return (pending: true, nil) } guard let data = result?.0 else { @@ -360,7 +413,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { throw BasicError.message(messageText: errorMessage) } - return audioUrl + return (pending: false, url: audioUrl) } public func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully _: Bool) {