Generate a queue of player items when speech synthesis starts

This should better handle cases where playback catches up to
audio synthesis/download, and will have the player items put
into a wait state.

This does make seeking a little less performant, as AVPlayer
wont aggresively pre-load as we were before.

apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift

@@ -52,21 +52,30 @@ let VOICES = [
   VoicePair(firstKey: "en-SG-LunaNeural", secondKey: "en-SG-WayneNeural", firstName: "Luna (Singapore)", secondName: "Wayne (Singapore)")
 ]
 
+// Somewhat based on: https://github.com/neekeetab/CachingPlayerItem/blob/master/CachingPlayerItem.swift
 class SpeechPlayerItem: AVPlayerItem {
+  let resourceLoaderDelegate = ResourceLoaderDelegate()
   let session: AudioController
   let speechItem: SpeechItem
   let completed: () -> Void
 
   var observer: Any?
 
-  init(session: AudioController, speechItem: SpeechItem, url: URL, completed: @escaping () -> Void) {
-    self.session = session
+  init(session: AudioController, speechItem: SpeechItem, completed: @escaping () -> Void) {
     self.speechItem = speechItem
+    self.session = session
     self.completed = completed
 
-    let asset = AVAsset(url: url)
+    guard let fakeUrl = URL(string: "app.omnivore.speech://\(speechItem.localAudioURL.path).mp3") else {
+      fatalError("internal inconsistency")
+    }
+
+    let asset = AVURLAsset(url: fakeUrl)
+    asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main)
+
     super.init(asset: asset, automaticallyLoadedAssetKeys: nil)
-    session.updateDuration(forItem: speechItem, newDuration: CMTimeGetSeconds(asset.duration))
+
+    resourceLoaderDelegate.owner = self
 
     self.observer = observe(\.status, options: [.new]) { item, _ in
       item.session.updateDuration(forItem: item.speechItem, newDuration: CMTimeGetSeconds(item.duration))
@@ -76,6 +85,116 @@ class SpeechPlayerItem: AVPlayerItem {
       self.completed()
     }
   }
+
+  deinit {
+    NotificationCenter.default.removeObserver(self)
+    removeObserver(self, forKeyPath: "status")
+    resourceLoaderDelegate.session?.invalidateAndCancel()
+  }
+
+  open func download() {
+    if resourceLoaderDelegate.session == nil {
+      resourceLoaderDelegate.startDataRequest(with: speechItem.urlRequest)
+    }
+  }
+
+  @objc func playbackStalledHandler() {
+    print("playback stalled...")
+  }
+
+  class ResourceLoaderDelegate: NSObject, AVAssetResourceLoaderDelegate {
+    var session: URLSession?
+    var mediaData: Data?
+    var pendingRequests = Set<AVAssetResourceLoadingRequest>()
+    weak var owner: SpeechPlayerItem?
+
+    func resourceLoader(_: AVAssetResourceLoader, shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest) -> Bool {
+      if session == nil {
+        guard let initialUrl = owner?.speechItem.urlRequest else {
+          fatalError("internal inconsistency")
+        }
+
+        startDataRequest(with: initialUrl)
+      }
+
+      pendingRequests.insert(loadingRequest)
+      processPendingRequests()
+      return true
+    }
+
+    func startDataRequest(with _: URLRequest) {
+      let configuration = URLSessionConfiguration.default
+      configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData
+      session = URLSession(configuration: configuration)
+
+      Task {
+        guard let speechItem = self.owner?.speechItem else {
+          // This probably can't happen, but if it does, just returning should
+          // let AVPlayer try again.
+          print("No speech item found: ", self.owner)
+          return
+        }
+
+        // TODO: how do we want to propogate this and handle it in the player
+        // The exception is just from some old code and does nothing.
+        let audioData = try? await SpeechSynthesizer.download(speechItem: speechItem, session: self.session)
+        DispatchQueue.main.async {
+          self.mediaData = audioData
+          self.processPendingRequests()
+        }
+      }
+    }
+
+    func resourceLoader(_: AVAssetResourceLoader, didCancel loadingRequest: AVAssetResourceLoadingRequest) {
+      pendingRequests.remove(loadingRequest)
+    }
+
+    func processPendingRequests() {
+      let requestsFulfilled = Set<AVAssetResourceLoadingRequest>(pendingRequests.compactMap {
+        self.fillInContentInformationRequest($0.contentInformationRequest)
+        if self.haveEnoughDataToFulfillRequest($0.dataRequest!) {
+          $0.finishLoading()
+          return $0
+        }
+        return nil
+      })
+
+      // remove fulfilled requests from pending requests
+      _ = requestsFulfilled.map { self.pendingRequests.remove($0) }
+    }
+
+    func fillInContentInformationRequest(_ contentInformationRequest: AVAssetResourceLoadingContentInformationRequest?) {
+      contentInformationRequest?.contentType = UTType.mp3.identifier
+
+      if let mediaData = mediaData {
+        contentInformationRequest?.isByteRangeAccessSupported = true
+        contentInformationRequest?.contentLength = Int64(mediaData.count)
+      }
+    }
+
+    func haveEnoughDataToFulfillRequest(_ dataRequest: AVAssetResourceLoadingDataRequest) -> Bool {
+      let requestedOffset = Int(dataRequest.requestedOffset)
+      let requestedLength = dataRequest.requestedLength
+      let currentOffset = Int(dataRequest.currentOffset)
+
+      guard let songDataUnwrapped = mediaData,
+            songDataUnwrapped.count > currentOffset
+      else {
+        // Don't have any data at all for this request.
+        return false
+      }
+
+      let bytesToRespond = min(songDataUnwrapped.count - currentOffset, requestedLength)
+      let dataToRespond = songDataUnwrapped.subdata(in: Range(uncheckedBounds: (currentOffset, currentOffset + bytesToRespond)))
+      dataRequest.respond(with: dataToRespond)
+
+      return songDataUnwrapped.count >= requestedLength + requestedOffset
+    }
+
+    deinit {
+      session?.invalidateAndCancel()
+    }
+  }
 }
 
 public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate {
@@ -147,15 +266,30 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
   public func preload(itemIDs: [String], retryCount _: Int = 0) async -> Bool {
     for itemID in itemIDs {
       print("preloading speech file: ", itemID)
-      _ = try? await downloadSpeechFile(itemID: itemID, priority: .low)
+      if let document = try? await downloadSpeechFile(itemID: itemID, priority: .low) {
+        let synthesizer = SpeechSynthesizer(appEnvironment: appEnvironment, networker: networker, document: document)
+        do {
+          try await synthesizer.preload()
+          return true
+        } catch {
+          print("error preloading audio file", error)
+        }
+      }
     }
-    return true
+    return false
   }
 
   public func downloadForOffline(itemID: String) async -> Bool {
     if let document = try? await downloadSpeechFile(itemID: itemID, priority: .low) {
       let synthesizer = SpeechSynthesizer(appEnvironment: appEnvironment, networker: networker, document: document)
-      for await _ in synthesizer.fetch(from: 0) {}
+      for item in synthesizer.createPlayerItems(from: 0) {
+        do {
+          _ = try await SpeechSynthesizer.download(speechItem: item)
+        } catch {
+          print("error downloading audio segment: ", error)
+          return false
+        }
+      }
       return true
     }
     return false
@@ -373,35 +507,58 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
   }
 
   func synthesizeFrom(start: Int, playWhenReady: Bool, atOffset: Double = 0.0) {
-    playbackTask = Task {
-      if let synthesizer = synthesizer {
-        for await speechItem in synthesizer.fetch(from: start) {
-          DispatchQueue.main.async {
-            let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1
-            let item = SpeechPlayerItem(session: self, speechItem: speechItem, url: speechItem.audioURL) {
-              // Pause player when we complete the final item.
-              if isLast {
-                self.player?.pause()
-                self.state = .reachedEnd
-              }
-            }
-            self.player?.insert(item, after: nil)
-
-            if playWhenReady, self.player?.items().count == 1 {
-              if atOffset > 0.0 {
-                item.seek(to: CMTimeMakeWithSeconds(atOffset, preferredTimescale: 600)) { success in
-                  print("success seeking to time: ", success)
-                  self.fireTimer()
-                }
-              }
-              self.startTimer()
-              self.unpause()
-              self.setupRemoteControl()
+    if let synthesizer = self.synthesizer, let items = self.synthesizer?.createPlayerItems(from: start) {
+      for speechItem in items {
+        let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1
+        let playerItem = SpeechPlayerItem(session: self, speechItem: speechItem) {
+          if isLast {
+            self.player?.pause()
+            self.state = .reachedEnd
+          }
+        }
+        player?.insert(playerItem, after: nil)
+        if playWhenReady, player?.items().count == 1 {
+          if atOffset > 0.0 {
+            playerItem.seek(to: CMTimeMakeWithSeconds(atOffset, preferredTimescale: 600)) { success in
+              print("success seeking to time: ", success)
+              self.fireTimer()
             }
           }
+          startTimer()
+          unpause()
+          setupRemoteControl()
         }
       }
     }
+//    playbackTask = Task {
+//      if let synthesizer = synthesizer {
+//        for await speechItem in synthesizer.fetch(from: start) {
+//          DispatchQueue.main.async {
+//            let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1
+//            let item = SpeechPlayerItem(session: self, speechItem: speechItem, url: speechItem.localAudioURL) {
+//              // Pause player when we complete the final item.
+//              if isLast {
+//                self.player?.pause()
+//                self.state = .reachedEnd
+//              }
+//            }
+//            self.player?.insert(item, after: nil)
+//
+//            if playWhenReady, self.player?.items().count == 1 {
+//              if atOffset > 0.0 {
+//                item.seek(to: CMTimeMakeWithSeconds(atOffset, preferredTimescale: 600)) { success in
+//                  print("success seeking to time: ", success)
+//                  self.fireTimer()
+//                }
+//              }
+//              self.startTimer()
+//              self.unpause()
+//              self.setupRemoteControl()
+//            }
+//          }
+//        }
+//      }
+//    }
   }
 
   public func pause() {
@@ -444,7 +601,6 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
   @objc func fireTimer() {
     if let player = player {
       if player.error != nil || player.currentItem?.error != nil {
-        print("ERROR IN PLAYBACK")
         stop()
       }
 

apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift

@@ -55,7 +55,8 @@ struct SpeechDocument: Decodable {
 struct SpeechItem {
   let htmlIdx: String
   let audioIdx: Int
-  let audioURL: URL
+  let urlRequest: URLRequest
+  let localAudioURL: URL
 }
 
 struct SpeechSynthesizer {
@@ -74,62 +75,155 @@ struct SpeechSynthesizer {
     document.utterances.map { document.estimatedDuration(utterance: $0, speed: speed) }
   }
 
-  func fetch(from: Int) -> SpeechSynthesisFetcher {
-    SpeechSynthesisFetcher(synthesizer: self, start: from)
-  }
-}
-
-struct SpeechSynthesisFetcher: AsyncSequence {
-  typealias Element = SpeechItem
-  let start: Int
-  let synthesizer: SpeechSynthesizer
-
-  init(synthesizer: SpeechSynthesizer, start: Int) {
-    self.start = start
-    self.synthesizer = synthesizer
+  func preload() async throws {
+    if let item = speechItemForIdx(idx: 0) {
+      _ = try await Self.download(speechItem: item)
+    }
   }
 
-  func makeAsyncIterator() -> SpeechSynthesizerIterator {
-    SpeechSynthesizerIterator(synthesizer: synthesizer, start: start)
-  }
+  func speechItemForIdx(idx: Int) -> SpeechItem? {
+    let utterance = document.utterances[idx]
+    let voiceStr = utterance.voice ?? document.defaultVoice
+    let segmentStr = String(format: "%04d", arguments: [idx])
+    let localAudioURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3")
 
-  struct SpeechSynthesizerIterator: AsyncIteratorProtocol {
-    let synthesizer: SpeechSynthesizer
-
-    init(synthesizer: SpeechSynthesizer, start: Int) {
-      self.synthesizer = synthesizer
-      self.currentIdx = start
+    if let request = urlRequestFor(utterance: utterance) {
+      let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: idx, urlRequest: request, localAudioURL: localAudioURL)
+      return item
     }
 
-    var currentIdx: Int
+    return nil
+  }
 
-    mutating func next() async -> SpeechItem? {
-      if Task.isCancelled {
-        return nil
+  func createPlayerItems(from: Int) -> [SpeechItem] {
+    var result: [SpeechItem] = []
+
+    for idx in from ..< document.utterances.count {
+      let utterance = document.utterances[idx]
+      let voiceStr = utterance.voice ?? document.defaultVoice
+      let segmentStr = String(format: "%04d", arguments: [idx])
+      let localAudioURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3")
+
+      if let request = urlRequestFor(utterance: utterance) {
+        let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: idx, urlRequest: request, localAudioURL: localAudioURL)
+        result.append(item)
+      } else {
+        // TODO: How do we want to handle completely skipped paragraphs?
+      }
+    }
+
+    return result
+  }
+
+  func urlRequestFor(utterance: Utterance) -> URLRequest? {
+    var request = URLRequest(url: appEnvironment.ttsBaseURL)
+    request.httpMethod = "POST"
+    request.timeoutInterval = 600
+
+    if let ssml = try? utterance.toSSML(document: document) {
+      request.httpBody = ssml
+    }
+
+    for (header, value) in networker.defaultHeaders {
+      request.setValue(value, forHTTPHeaderField: header)
+    }
+
+    return request
+  }
+
+  static func download(speechItem: SpeechItem, redownloadCached: Bool = false, session: URLSession? = URLSession.shared) async throws -> Data? {
+    if !redownloadCached, FileManager.default.fileExists(atPath: speechItem.localAudioURL.path) {
+      if let localData = try? Data(contentsOf: speechItem.localAudioURL) {
+        return localData
+      }
+    }
+
+    let request = speechItem.urlRequest
+    let result: (Data, URLResponse)? = try? await (session ?? URLSession.shared).data(for: request)
+    guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else {
+      print("error: ", result?.1 as Any)
+      throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.")
+    }
+
+    guard let data = result?.0 else {
+      throw BasicError.message(messageText: "audioFetch failed. no data received.")
+    }
+
+    let tempPath = FileManager.default
+      .urls(for: .cachesDirectory, in: .userDomainMask)[0]
+      .appendingPathComponent(UUID().uuidString + ".mp3")
+
+    do {
+      let decoder = JSONDecoder()
+      let jsonData = try decoder.decode(SynthesizeResult.self, from: data)
+      let audioData = Data(fromHexEncodedString: jsonData.audioData)!
+      if audioData.count < 1 {
+        throw BasicError.message(messageText: "Audio data is empty")
       }
 
-      if currentIdx >= synthesizer.document.utterances.count {
-        return nil
-      }
+      try audioData.write(to: tempPath)
+      try? FileManager.default.removeItem(at: speechItem.localAudioURL)
+      try FileManager.default.moveItem(at: tempPath, to: speechItem.localAudioURL)
 
-      let utterance = synthesizer.document.utterances[currentIdx]
-      let fetched = try? await fetchUtterance(appEnvironment: synthesizer.appEnvironment,
-                                              networker: synthesizer.networker,
-                                              document: synthesizer.document,
-                                              segmentIdx: currentIdx,
-                                              utterance: utterance)
-
-      if let fetchedURL = fetched {
-        let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: currentIdx, audioURL: fetchedURL)
-        currentIdx += 1
-        return item
-      }
-
-      return nil
+      return audioData
+    } catch {
+      let errorMessage = "audioFetch failed. could not write MP3 data to disk"
+      throw BasicError.message(messageText: errorMessage)
     }
   }
 }
 
+// struct SpeechSynthesisFetcher: AsyncSequence {
+//  typealias Element = SpeechItem
+//  let start: Int
+//  let synthesizer: SpeechSynthesizer
+//
+//  init(synthesizer: SpeechSynthesizer, start: Int) {
+//    self.start = start
+//    self.synthesizer = synthesizer
+//  }
+//
+//  func makeAsyncIterator() -> SpeechSynthesizerIterator {
+//    SpeechSynthesizerIterator(synthesizer: synthesizer, start: start)
+//  }
+//
+//  struct SpeechSynthesizerIterator: AsyncIteratorProtocol {
+//    let synthesizer: SpeechSynthesizer
+//
+//    init(synthesizer: SpeechSynthesizer, start: Int) {
+//      self.synthesizer = synthesizer
+//      self.currentIdx = start
+//    }
+//
+//    var currentIdx: Int
+//
+//    mutating func next() async -> SpeechItem? {
+//      if Task.isCancelled {
+//        return nil
+//      }
+//
+//      if currentIdx >= synthesizer.document.utterances.count {
+//        return nil
+//      }
+//
+//      let utterance = synthesizer.document.utterances[currentIdx]
+//      let fetched = try? await fetchUtterance(appEnvironment: synthesizer.appEnvironment,
+//                                              networker: synthesizer.networker,
+//                                              document: synthesizer.document,
+//                                              segmentIdx: currentIdx,
+//                                              utterance: utterance)
+//
+//      if let fetchedURL = fetched {
+//        let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: currentIdx, audioURL: fetchedURL)
+//        currentIdx += 1
+//        return item
+//      }
+//
+//      return nil
+//    }
+//  }
+// }
+
 struct SynthesizeResult: Decodable {
   let audioData: String
 //  let speechMarks: Any?