diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift index 269ab2096..cc8b2839b 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift @@ -52,21 +52,30 @@ let VOICES = [ VoicePair(firstKey: "en-SG-LunaNeural", secondKey: "en-SG-WayneNeural", firstName: "Luna (Singapore)", secondName: "Wayne (Singapore)") ] +// Somewhat based on: https://github.com/neekeetab/CachingPlayerItem/blob/master/CachingPlayerItem.swift class SpeechPlayerItem: AVPlayerItem { + let resourceLoaderDelegate = ResourceLoaderDelegate() let session: AudioController let speechItem: SpeechItem let completed: () -> Void var observer: Any? - init(session: AudioController, speechItem: SpeechItem, url: URL, completed: @escaping () -> Void) { - self.session = session + init(session: AudioController, speechItem: SpeechItem, completed: @escaping () -> Void) { self.speechItem = speechItem + self.session = session self.completed = completed - let asset = AVAsset(url: url) + guard let fakeUrl = URL(string: "app.omnivore.speech://\(speechItem.localAudioURL.path).mp3") else { + fatalError("internal inconsistency") + } + + let asset = AVURLAsset(url: fakeUrl) + asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main) + super.init(asset: asset, automaticallyLoadedAssetKeys: nil) - session.updateDuration(forItem: speechItem, newDuration: CMTimeGetSeconds(asset.duration)) + + resourceLoaderDelegate.owner = self self.observer = observe(\.status, options: [.new]) { item, _ in item.session.updateDuration(forItem: item.speechItem, newDuration: CMTimeGetSeconds(item.duration)) @@ -76,6 +85,116 @@ class SpeechPlayerItem: AVPlayerItem { self.completed() } } + + deinit { + NotificationCenter.default.removeObserver(self) + removeObserver(self, forKeyPath: "status") + resourceLoaderDelegate.session?.invalidateAndCancel() + } + + open func download() { + if resourceLoaderDelegate.session == nil { + resourceLoaderDelegate.startDataRequest(with: speechItem.urlRequest) + } + } + + @objc func playbackStalledHandler() { + print("playback stalled...") + } + + class ResourceLoaderDelegate: NSObject, AVAssetResourceLoaderDelegate { + var session: URLSession? + var mediaData: Data? + var pendingRequests = Set() + weak var owner: SpeechPlayerItem? + + func resourceLoader(_: AVAssetResourceLoader, shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest) -> Bool { + if session == nil { + guard let initialUrl = owner?.speechItem.urlRequest else { + fatalError("internal inconsistency") + } + + startDataRequest(with: initialUrl) + } + + pendingRequests.insert(loadingRequest) + processPendingRequests() + return true + } + + func startDataRequest(with _: URLRequest) { + let configuration = URLSessionConfiguration.default + configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData + session = URLSession(configuration: configuration) + + Task { + guard let speechItem = self.owner?.speechItem else { + // This probably can't happen, but if it does, just returning should + // let AVPlayer try again. + print("No speech item found: ", self.owner) + return + } + + // TODO: how do we want to propogate this and handle it in the player + // The exception is just from some old code and does nothing. + let audioData = try? await SpeechSynthesizer.download(speechItem: speechItem, session: self.session) + DispatchQueue.main.async { + self.mediaData = audioData + self.processPendingRequests() + } + } + } + + func resourceLoader(_: AVAssetResourceLoader, didCancel loadingRequest: AVAssetResourceLoadingRequest) { + pendingRequests.remove(loadingRequest) + } + + func processPendingRequests() { + let requestsFulfilled = Set(pendingRequests.compactMap { + self.fillInContentInformationRequest($0.contentInformationRequest) + if self.haveEnoughDataToFulfillRequest($0.dataRequest!) { + $0.finishLoading() + return $0 + } + return nil + }) + + // remove fulfilled requests from pending requests + _ = requestsFulfilled.map { self.pendingRequests.remove($0) } + } + + func fillInContentInformationRequest(_ contentInformationRequest: AVAssetResourceLoadingContentInformationRequest?) { + contentInformationRequest?.contentType = UTType.mp3.identifier + + if let mediaData = mediaData { + contentInformationRequest?.isByteRangeAccessSupported = true + contentInformationRequest?.contentLength = Int64(mediaData.count) + } + } + + func haveEnoughDataToFulfillRequest(_ dataRequest: AVAssetResourceLoadingDataRequest) -> Bool { + let requestedOffset = Int(dataRequest.requestedOffset) + let requestedLength = dataRequest.requestedLength + let currentOffset = Int(dataRequest.currentOffset) + + guard let songDataUnwrapped = mediaData, + songDataUnwrapped.count > currentOffset + else { + // Don't have any data at all for this request. + return false + } + + let bytesToRespond = min(songDataUnwrapped.count - currentOffset, requestedLength) + let dataToRespond = songDataUnwrapped.subdata(in: Range(uncheckedBounds: (currentOffset, currentOffset + bytesToRespond))) + dataRequest.respond(with: dataToRespond) + + return songDataUnwrapped.count >= requestedLength + requestedOffset + } + + deinit { + session?.invalidateAndCancel() + } + } } public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate { @@ -147,15 +266,30 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate public func preload(itemIDs: [String], retryCount _: Int = 0) async -> Bool { for itemID in itemIDs { print("preloading speech file: ", itemID) - _ = try? await downloadSpeechFile(itemID: itemID, priority: .low) + if let document = try? await downloadSpeechFile(itemID: itemID, priority: .low) { + let synthesizer = SpeechSynthesizer(appEnvironment: appEnvironment, networker: networker, document: document) + do { + try await synthesizer.preload() + return true + } catch { + print("error preloading audio file", error) + } + } } - return true + return false } public func downloadForOffline(itemID: String) async -> Bool { if let document = try? await downloadSpeechFile(itemID: itemID, priority: .low) { let synthesizer = SpeechSynthesizer(appEnvironment: appEnvironment, networker: networker, document: document) - for await _ in synthesizer.fetch(from: 0) {} + for item in synthesizer.createPlayerItems(from: 0) { + do { + _ = try await SpeechSynthesizer.download(speechItem: item) + } catch { + print("error downloading audio segment: ", error) + return false + } + } return true } return false @@ -373,35 +507,58 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate } func synthesizeFrom(start: Int, playWhenReady: Bool, atOffset: Double = 0.0) { - playbackTask = Task { - if let synthesizer = synthesizer { - for await speechItem in synthesizer.fetch(from: start) { - DispatchQueue.main.async { - let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1 - let item = SpeechPlayerItem(session: self, speechItem: speechItem, url: speechItem.audioURL) { - // Pause player when we complete the final item. - if isLast { - self.player?.pause() - self.state = .reachedEnd - } - } - self.player?.insert(item, after: nil) - - if playWhenReady, self.player?.items().count == 1 { - if atOffset > 0.0 { - item.seek(to: CMTimeMakeWithSeconds(atOffset, preferredTimescale: 600)) { success in - print("success seeking to time: ", success) - self.fireTimer() - } - } - self.startTimer() - self.unpause() - self.setupRemoteControl() + if let synthesizer = self.synthesizer, let items = self.synthesizer?.createPlayerItems(from: start) { + for speechItem in items { + let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1 + let playerItem = SpeechPlayerItem(session: self, speechItem: speechItem) { + if isLast { + self.player?.pause() + self.state = .reachedEnd + } + } + player?.insert(playerItem, after: nil) + if playWhenReady, player?.items().count == 1 { + if atOffset > 0.0 { + playerItem.seek(to: CMTimeMakeWithSeconds(atOffset, preferredTimescale: 600)) { success in + print("success seeking to time: ", success) + self.fireTimer() } } + startTimer() + unpause() + setupRemoteControl() } } } +// playbackTask = Task { +// if let synthesizer = synthesizer { +// for await speechItem in synthesizer.fetch(from: start) { +// DispatchQueue.main.async { +// let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1 +// let item = SpeechPlayerItem(session: self, speechItem: speechItem, url: speechItem.localAudioURL) { +// // Pause player when we complete the final item. +// if isLast { +// self.player?.pause() +// self.state = .reachedEnd +// } +// } +// self.player?.insert(item, after: nil) +// +// if playWhenReady, self.player?.items().count == 1 { +// if atOffset > 0.0 { +// item.seek(to: CMTimeMakeWithSeconds(atOffset, preferredTimescale: 600)) { success in +// print("success seeking to time: ", success) +// self.fireTimer() +// } +// } +// self.startTimer() +// self.unpause() +// self.setupRemoteControl() +// } +// } +// } +// } +// } } public func pause() { @@ -444,7 +601,6 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate @objc func fireTimer() { if let player = player { if player.error != nil || player.currentItem?.error != nil { - print("ERROR IN PLAYBACK") stop() } diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift index 290f0b483..7f615db90 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift @@ -55,7 +55,8 @@ struct SpeechDocument: Decodable { struct SpeechItem { let htmlIdx: String let audioIdx: Int - let audioURL: URL + let urlRequest: URLRequest + let localAudioURL: URL } struct SpeechSynthesizer { @@ -74,62 +75,155 @@ struct SpeechSynthesizer { document.utterances.map { document.estimatedDuration(utterance: $0, speed: speed) } } - func fetch(from: Int) -> SpeechSynthesisFetcher { - SpeechSynthesisFetcher(synthesizer: self, start: from) - } -} - -struct SpeechSynthesisFetcher: AsyncSequence { - typealias Element = SpeechItem - let start: Int - let synthesizer: SpeechSynthesizer - - init(synthesizer: SpeechSynthesizer, start: Int) { - self.start = start - self.synthesizer = synthesizer + func preload() async throws { + if let item = speechItemForIdx(idx: 0) { + _ = try await Self.download(speechItem: item) + } } - func makeAsyncIterator() -> SpeechSynthesizerIterator { - SpeechSynthesizerIterator(synthesizer: synthesizer, start: start) - } + func speechItemForIdx(idx: Int) -> SpeechItem? { + let utterance = document.utterances[idx] + let voiceStr = utterance.voice ?? document.defaultVoice + let segmentStr = String(format: "%04d", arguments: [idx]) + let localAudioURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3") - struct SpeechSynthesizerIterator: AsyncIteratorProtocol { - let synthesizer: SpeechSynthesizer - - init(synthesizer: SpeechSynthesizer, start: Int) { - self.synthesizer = synthesizer - self.currentIdx = start + if let request = urlRequestFor(utterance: utterance) { + let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: idx, urlRequest: request, localAudioURL: localAudioURL) + return item } - var currentIdx: Int + return nil + } - mutating func next() async -> SpeechItem? { - if Task.isCancelled { - return nil + func createPlayerItems(from: Int) -> [SpeechItem] { + var result: [SpeechItem] = [] + + for idx in from ..< document.utterances.count { + let utterance = document.utterances[idx] + let voiceStr = utterance.voice ?? document.defaultVoice + let segmentStr = String(format: "%04d", arguments: [idx]) + let localAudioURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3") + + if let request = urlRequestFor(utterance: utterance) { + let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: idx, urlRequest: request, localAudioURL: localAudioURL) + result.append(item) + } else { + // TODO: How do we want to handle completely skipped paragraphs? + } + } + + return result + } + + func urlRequestFor(utterance: Utterance) -> URLRequest? { + var request = URLRequest(url: appEnvironment.ttsBaseURL) + request.httpMethod = "POST" + request.timeoutInterval = 600 + + if let ssml = try? utterance.toSSML(document: document) { + request.httpBody = ssml + } + + for (header, value) in networker.defaultHeaders { + request.setValue(value, forHTTPHeaderField: header) + } + + return request + } + + static func download(speechItem: SpeechItem, redownloadCached: Bool = false, session: URLSession? = URLSession.shared) async throws -> Data? { + if !redownloadCached, FileManager.default.fileExists(atPath: speechItem.localAudioURL.path) { + if let localData = try? Data(contentsOf: speechItem.localAudioURL) { + return localData + } + } + + let request = speechItem.urlRequest + let result: (Data, URLResponse)? = try? await (session ?? URLSession.shared).data(for: request) + guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else { + print("error: ", result?.1 as Any) + throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.") + } + + guard let data = result?.0 else { + throw BasicError.message(messageText: "audioFetch failed. no data received.") + } + + let tempPath = FileManager.default + .urls(for: .cachesDirectory, in: .userDomainMask)[0] + .appendingPathComponent(UUID().uuidString + ".mp3") + + do { + let decoder = JSONDecoder() + let jsonData = try decoder.decode(SynthesizeResult.self, from: data) + let audioData = Data(fromHexEncodedString: jsonData.audioData)! + if audioData.count < 1 { + throw BasicError.message(messageText: "Audio data is empty") } - if currentIdx >= synthesizer.document.utterances.count { - return nil - } + try audioData.write(to: tempPath) + try? FileManager.default.removeItem(at: speechItem.localAudioURL) + try FileManager.default.moveItem(at: tempPath, to: speechItem.localAudioURL) - let utterance = synthesizer.document.utterances[currentIdx] - let fetched = try? await fetchUtterance(appEnvironment: synthesizer.appEnvironment, - networker: synthesizer.networker, - document: synthesizer.document, - segmentIdx: currentIdx, - utterance: utterance) - - if let fetchedURL = fetched { - let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: currentIdx, audioURL: fetchedURL) - currentIdx += 1 - return item - } - - return nil + return audioData + } catch { + let errorMessage = "audioFetch failed. could not write MP3 data to disk" + throw BasicError.message(messageText: errorMessage) } } } +// struct SpeechSynthesisFetcher: AsyncSequence { +// typealias Element = SpeechItem +// let start: Int +// let synthesizer: SpeechSynthesizer +// +// init(synthesizer: SpeechSynthesizer, start: Int) { +// self.start = start +// self.synthesizer = synthesizer +// } +// +// func makeAsyncIterator() -> SpeechSynthesizerIterator { +// SpeechSynthesizerIterator(synthesizer: synthesizer, start: start) +// } +// +// struct SpeechSynthesizerIterator: AsyncIteratorProtocol { +// let synthesizer: SpeechSynthesizer +// +// init(synthesizer: SpeechSynthesizer, start: Int) { +// self.synthesizer = synthesizer +// self.currentIdx = start +// } +// +// var currentIdx: Int +// +// mutating func next() async -> SpeechItem? { +// if Task.isCancelled { +// return nil +// } +// +// if currentIdx >= synthesizer.document.utterances.count { +// return nil +// } +// +// let utterance = synthesizer.document.utterances[currentIdx] +// let fetched = try? await fetchUtterance(appEnvironment: synthesizer.appEnvironment, +// networker: synthesizer.networker, +// document: synthesizer.document, +// segmentIdx: currentIdx, +// utterance: utterance) +// +// if let fetchedURL = fetched { +// let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: currentIdx, audioURL: fetchedURL) +// currentIdx += 1 +// return item +// } +// +// return nil +// } +// } +// } + struct SynthesizeResult: Decodable { let audioData: String // let speechMarks: Any?