WIP: add CachingPlayerItem and switch to AVPlayer for streaming audio

This commit is contained in:
Jackson Harper 2022-09-01 20:50:10 +08:00
parent 4d34142d9d
commit 0a5e02ec6b
3 changed files with 383 additions and 70 deletions

View file

@ -33,11 +33,7 @@ struct WelcomeView: View {
var headlineText: some View {
Group {
if horizontalSizeClass == .compact {
Text("Everything you read. Safe, organized, and easy to share.")
} else {
Text("Everything you read. Safe,\norganized, and easy to share.")
}
Text("Never miss a great read.")
}
.font(.appLargeTitle)
}

View file

@ -36,7 +36,7 @@ enum DownloadPriority: String {
}
// Our observable object class
public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate, CachingPlayerItemDelegate {
@Published public var state: AudioSessionState = .stopped
@Published public var item: LinkedItem?
@ -49,7 +49,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
let networker: Networker
var timer: Timer?
var player: AVAudioPlayer?
var player: AVPlayer?
var downloadTask: Task<Void, Error>?
public init(appEnvironment: AppEnvironment, networker: Networker) {
@ -65,7 +65,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
}
public func stop() {
player?.stop()
// player?.stop()
clearNowPlayingInfo()
timer = nil
player = nil
@ -134,11 +134,16 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
case .scrubStarted:
return
case let .scrubEnded(seekTime):
player?.currentTime = seekTime
seek(to: seekTime)
// player?.currentTime = seekTime
}
}
}
public func seek(to _: TimeInterval) {
// seek
}
public var currentVoice: String {
"en-US-JennyNeural"
}
@ -151,16 +156,19 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
state == .playing && self.item == item
}
public func skipForward(seconds: Double) {
if let current = player?.currentTime {
player?.currentTime = min(duration, current + seconds)
}
public func skipForward(seconds _: Double) {
// if let current = player?.currentTime {
// seek(to: current + seconds)
//// player?.currentTime = min(duration, current + seconds)
// }
}
public func skipBackwards(seconds: Double) {
if let current = player?.currentTime {
player?.currentTime = max(0, current - seconds)
}
public func skipBackwards(seconds _: Double) {
// if let current = player?.currentTime {
// seek(to: current + seconds)
//
//// player?.currentTime = max(0, current - seconds)
// }
}
public func fileNameForAudioFile(_ pageId: String) -> String {
@ -180,21 +188,21 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
let pageId = item!.unwrappedID
downloadTask = Task {
let result = try? await downloadAudioFile(pageId: pageId, type: .mp3, priority: .high)
if Task.isCancelled { return }
if result == nil {
DispatchQueue.main.async {
NSNotification.operationSuccess(message: "Error generating audio.")
self.stop()
}
}
if let result = result, result.pending {
DispatchQueue.main.async {
NSNotification.operationSuccess(message: "Your audio is being generated.")
}
}
// let result = try? await downloadAudioFile(pageId: pageId, type: .mp3, priority: .high)
// if Task.isCancelled { return }
//
// if result == nil {
// DispatchQueue.main.async {
// NSNotification.operationSuccess(message: "Error generating audio.")
// self.stop()
// }
// }
//
// if let result = result, result.pending {
// DispatchQueue.main.async {
// NSNotification.operationSuccess(message: "Your audio is being generated.")
// }
// }
DispatchQueue.main.async {
self.startDownloadedAudioFile(pageId: pageId)
@ -208,28 +216,44 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
state = .stopped
return
}
// TODO: Maybe check if app is active so it doesn't end up playing later?
let audioUrl = pathForAudioFile(pageId: pageId)
if !FileManager.default.fileExists(atPath: audioUrl.path) {
stop()
return
}
//
// // TODO: Maybe check if app is active so it doesn't end up playing later?
//
// let audioUrl = pathForAudioFile(pageId: pageId)
// if !FileManager.default.fileExists(atPath: audioUrl.path) {
// stop()
// return
// }
do {
try AVAudioSession.sharedInstance().setCategory(.playback)
try AVAudioSession.sharedInstance().setCategory(.playback, mode: .default, options: [])
player = try AVAudioPlayer(contentsOf: audioUrl)
player?.delegate = self
if player?.play() ?? false {
state = .playing
startTimer()
setupRemoteControl()
}
let token = ValetKey.authToken.value()!
let url = URL(string: "https://text-to-speech-streaming-bryle2uxwq-wl.a.run.app/?token=\(token)&q=1")!
print("LOADING URL: ", url)
// let url = URL(string: "https://storage.googleapis.com/omnivore-demo-files/speech/062bfcc2-8d59-4880-8a67-fe6ee739c510.mp3")!
// let url = URL(string: "http://devimages.apple.com/iphone/samples/bipbop/bipbopall.m3u8")!
let playerItem = CachingPlayerItem(url: url)
playerItem.delegate = self
player = AVPlayer(playerItem: playerItem)
print("created player: ", player, player?.error)
player?.automaticallyWaitsToMinimizeStalling = false
player?.play()
print("starting playing: ", player, player?.error)
//
// player = try AVAudioPlayer(contentsOf: audioUrl)
// player?.delegate = self
// if player?.play() ?? false {
state = .playing
startTimer()
// setupRemoteControl()
// }
} catch {
print("error playing MP3 file", error)
try? FileManager.default.removeItem(atPath: audioUrl.path)
// try? FileManager.default.removeItem(atPath: audioUrl.path)
state = .stopped
}
}
@ -259,7 +283,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
func startTimer() {
if timer == nil {
// Update every 100ms
timer = Timer.scheduledTimer(timeInterval: 0.1, target: self, selector: #selector(update(_:)), userInfo: nil, repeats: true)
timer = Timer.scheduledTimer(timeInterval: 10, target: self, selector: #selector(update(_:)), userInfo: nil, repeats: true)
timer?.fire()
}
}
@ -278,26 +302,37 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
// Every second, get the current playing time of the player and refresh the status of the player progressslider
@objc func update(_: Timer) {
if let player = player, player.isPlaying {
duration = player.duration
durationString = formatTimeInterval(duration)
switch scrubState {
case .reset:
timeElapsed = player.currentTime
timeElapsedString = formatTimeInterval(timeElapsed)
if var nowPlaying = MPNowPlayingInfoCenter.default().nowPlayingInfo {
nowPlaying[MPMediaItemPropertyPlaybackDuration] = NSNumber(value: duration)
nowPlaying[MPNowPlayingInfoPropertyElapsedPlaybackTime] = NSNumber(value: timeElapsed)
MPNowPlayingInfoCenter.default().nowPlayingInfo = nowPlaying
}
case .scrubStarted:
break
case let .scrubEnded(seekTime):
scrubState = .reset
timeElapsed = seekTime
}
if let player = player {
print("current error: ", player.error)
print("current state", player.rate)
print("current position", player.currentTime())
print("timeControlStatus", player.timeControlStatus.rawValue)
print("waiting Reason: ", player.reasonForWaitingToPlay?.rawValue)
print("currentItem.duration: ", player.currentItem?.duration)
print("status:", player.currentItem?.status.rawValue)
print("error:", player.currentItem?.error)
print("error log:", player.currentItem?.errorLog())
}
// if let player = player, player.isPlaying {
// duration = player.duration
// durationString = formatTimeInterval(duration)
//
// switch scrubState {
// case .reset:
// timeElapsed = player.currentTime
// timeElapsedString = formatTimeInterval(timeElapsed)
// if var nowPlaying = MPNowPlayingInfoCenter.default().nowPlayingInfo {
// nowPlaying[MPMediaItemPropertyPlaybackDuration] = NSNumber(value: duration)
// nowPlaying[MPNowPlayingInfoPropertyElapsedPlaybackTime] = NSNumber(value: timeElapsed)
// MPNowPlayingInfoCenter.default().nowPlayingInfo = nowPlaying
// }
// case .scrubStarted:
// break
// case let .scrubEnded(seekTime):
// scrubState = .reset
// timeElapsed = seekTime
// }
// }
}
func clearNowPlayingInfo() {
@ -353,7 +388,8 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
commandCenter.changePlaybackPositionCommand.isEnabled = true
commandCenter.changePlaybackPositionCommand.addTarget { event -> MPRemoteCommandHandlerStatus in
if let event = event as? MPChangePlaybackPositionCommandEvent {
self.player?.currentTime = event.positionTime
self.seek(to: event.positionTime)
// self.player?.currentTime = event.positionTime
return .success
}
return .commandFailed
@ -456,4 +492,31 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate {
default: ()
}
}
/// Is called when the media file is fully downloaded.
@objc func playerItem(_: CachingPlayerItem, didFinishDownloadingData data: Data) {
print("didFinishDownloadingData: ", data.underestimatedCount)
}
/// Is called every time a new portion of data is received.
@objc func playerItem(_: CachingPlayerItem, didDownloadBytesSoFar bytesDownloaded: Int, outOf _: Int) {
print("didDownloadBytesSoFar: ", bytesDownloaded)
}
/// Is called after initial prebuffering is finished, means
/// we are ready to play.
@objc func playerItemReadyToPlay(_: CachingPlayerItem) {
print("playerItemReadyToPlay")
}
/// Is called when the data being downloaded did not arrive in time to
/// continue playback.
@objc func playerItemPlaybackStalled(_: CachingPlayerItem) {
print("playerItemPlaybackStalled")
}
/// Is called on downloading error.
@objc func playerItem(_: CachingPlayerItem, downloadingFailedWith error: Error) {
print("downloadingFailedWith errpr", error)
}
}

View file

@ -0,0 +1,254 @@
// From: https://github.com/neekeetab/CachingPlayerItem
import AVFoundation
import Foundation
private extension URL {
func withScheme(_ scheme: String) -> URL? {
var components = URLComponents(url: self, resolvingAgainstBaseURL: false)
components?.scheme = scheme
return components?.url
}
}
@objc protocol CachingPlayerItemDelegate {
/// Is called when the media file is fully downloaded.
@objc optional func playerItem(_ playerItem: CachingPlayerItem, didFinishDownloadingData data: Data)
/// Is called every time a new portion of data is received.
@objc optional func playerItem(_ playerItem: CachingPlayerItem, didDownloadBytesSoFar bytesDownloaded: Int, outOf bytesExpected: Int)
/// Is called after initial prebuffering is finished, means
/// we are ready to play.
@objc optional func playerItemReadyToPlay(_ playerItem: CachingPlayerItem)
/// Is called when the data being downloaded did not arrive in time to
/// continue playback.
@objc optional func playerItemPlaybackStalled(_ playerItem: CachingPlayerItem)
/// Is called on downloading error.
@objc optional func playerItem(_ playerItem: CachingPlayerItem, downloadingFailedWith error: Error)
}
open class CachingPlayerItem: AVPlayerItem {
class ResourceLoaderDelegate: NSObject, AVAssetResourceLoaderDelegate, URLSessionDelegate, URLSessionDataDelegate, URLSessionTaskDelegate {
var playingFromData = false
var mimeType: String? // is required when playing from Data
var session: URLSession?
var mediaData: Data?
var response: URLResponse?
var pendingRequests = Set<AVAssetResourceLoadingRequest>()
weak var owner: CachingPlayerItem?
func resourceLoader(_: AVAssetResourceLoader, shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest) -> Bool {
if playingFromData {
// Nothing to load.
} else if session == nil {
// If we're playing from a url, we need to download the file.
// We start loading the file on first request only.
guard let initialUrl = owner?.url else {
fatalError("internal inconsistency")
}
startDataRequest(with: initialUrl)
}
pendingRequests.insert(loadingRequest)
processPendingRequests()
return true
}
func startDataRequest(with url: URL) {
let configuration = URLSessionConfiguration.default
configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData
session = URLSession(configuration: configuration, delegate: self, delegateQueue: nil)
session?.dataTask(with: url).resume()
}
func resourceLoader(_: AVAssetResourceLoader, didCancel loadingRequest: AVAssetResourceLoadingRequest) {
pendingRequests.remove(loadingRequest)
}
// MARK: URLSession delegate
func urlSession(_: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) {
mediaData?.append(data)
processPendingRequests()
owner?.delegate?.playerItem?(owner!, didDownloadBytesSoFar: mediaData!.count, outOf: Int(dataTask.countOfBytesExpectedToReceive))
}
func urlSession(_: URLSession, dataTask _: URLSessionDataTask, didReceive response: URLResponse, completionHandler: @escaping (URLSession.ResponseDisposition) -> Void) {
completionHandler(Foundation.URLSession.ResponseDisposition.allow)
mediaData = Data()
self.response = response
processPendingRequests()
}
func urlSession(_: URLSession, task _: URLSessionTask, didCompleteWithError error: Error?) {
if let errorUnwrapped = error {
owner?.delegate?.playerItem?(owner!, downloadingFailedWith: errorUnwrapped)
return
}
processPendingRequests()
owner?.delegate?.playerItem?(owner!, didFinishDownloadingData: mediaData!)
}
// MARK: -
func processPendingRequests() {
// get all fullfilled requests
let requestsFulfilled = Set<AVAssetResourceLoadingRequest>(pendingRequests.compactMap {
self.fillInContentInformationRequest($0.contentInformationRequest)
if self.haveEnoughDataToFulfillRequest($0.dataRequest!) {
$0.finishLoading()
return $0
}
return nil
})
// remove fulfilled requests from pending requests
_ = requestsFulfilled.map { self.pendingRequests.remove($0) }
}
func fillInContentInformationRequest(_ contentInformationRequest: AVAssetResourceLoadingContentInformationRequest?) {
// if we play from Data we make no url requests, therefore we have no responses, so we need to fill in contentInformationRequest manually
if playingFromData {
contentInformationRequest?.contentType = mimeType
contentInformationRequest?.contentLength = Int64(mediaData!.count)
contentInformationRequest?.isByteRangeAccessSupported = true
return
}
guard let responseUnwrapped = response else {
// have no response from the server yet
return
}
contentInformationRequest?.contentType = responseUnwrapped.mimeType
contentInformationRequest?.contentLength = responseUnwrapped.expectedContentLength
contentInformationRequest?.isByteRangeAccessSupported = true
}
func haveEnoughDataToFulfillRequest(_ dataRequest: AVAssetResourceLoadingDataRequest) -> Bool {
let requestedOffset = Int(dataRequest.requestedOffset)
let requestedLength = dataRequest.requestedLength
let currentOffset = Int(dataRequest.currentOffset)
guard let songDataUnwrapped = mediaData,
songDataUnwrapped.count > currentOffset
else {
// Don't have any data at all for this request.
return false
}
let bytesToRespond = min(songDataUnwrapped.count - currentOffset, requestedLength)
let dataToRespond = songDataUnwrapped.subdata(in: Range(uncheckedBounds: (currentOffset, currentOffset + bytesToRespond)))
dataRequest.respond(with: dataToRespond)
return songDataUnwrapped.count >= requestedLength + requestedOffset
}
deinit {
session?.invalidateAndCancel()
}
}
fileprivate let resourceLoaderDelegate = ResourceLoaderDelegate()
fileprivate let url: URL
fileprivate let initialScheme: String?
fileprivate var customFileExtension: String?
weak var delegate: CachingPlayerItemDelegate?
open func download() {
if resourceLoaderDelegate.session == nil {
resourceLoaderDelegate.startDataRequest(with: url)
}
}
private let cachingPlayerItemScheme = "cachingPlayerItemScheme"
/// Is used for playing remote files.
convenience init(url: URL) {
self.init(url: url, customFileExtension: nil)
}
/// Override/append custom file extension to URL path.
/// This is required for the player to work correctly with the intended file type.
init(url: URL, customFileExtension _: String?) {
guard let components = URLComponents(url: url, resolvingAgainstBaseURL: false),
let scheme = components.scheme,
var urlWithCustomScheme = url.withScheme(cachingPlayerItemScheme)
else {
fatalError("Urls without a scheme are not supported")
}
self.url = url
self.initialScheme = scheme
if let ext = customFileExtension {
urlWithCustomScheme.deletePathExtension()
urlWithCustomScheme.appendPathExtension(ext)
self.customFileExtension = ext
}
let mimeType = "audio/mpeg"
let asset = AVURLAsset(url: url, options: ["AVURLAssetOutOfBandMIMETypeKey": mimeType])
asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main)
super.init(asset: asset, automaticallyLoadedAssetKeys: nil)
resourceLoaderDelegate.owner = self
addObserver(self, forKeyPath: "status", options: NSKeyValueObservingOptions.new, context: nil)
NotificationCenter.default.addObserver(self, selector: #selector(playbackStalledHandler), name: NSNotification.Name.AVPlayerItemPlaybackStalled, object: self)
}
/// Is used for playing from Data.
init(data: Data, mimeType: String, fileExtension: String) {
guard let fakeUrl = URL(string: cachingPlayerItemScheme + "://whatever/file.\(fileExtension)") else {
fatalError("internal inconsistency")
}
self.url = fakeUrl
self.initialScheme = nil
resourceLoaderDelegate.mediaData = data
resourceLoaderDelegate.playingFromData = true
resourceLoaderDelegate.mimeType = mimeType
let asset = AVURLAsset(url: fakeUrl)
asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main)
super.init(asset: asset, automaticallyLoadedAssetKeys: nil)
resourceLoaderDelegate.owner = self
addObserver(self, forKeyPath: "status", options: NSKeyValueObservingOptions.new, context: nil)
NotificationCenter.default.addObserver(self, selector: #selector(playbackStalledHandler), name: NSNotification.Name.AVPlayerItemPlaybackStalled, object: self)
}
// MARK: KVO
override open func observeValue(forKeyPath _: String?, of _: Any?, change _: [NSKeyValueChangeKey: Any]?, context _: UnsafeMutableRawPointer?) {
delegate?.playerItemReadyToPlay?(self)
}
// MARK: Notification hanlers
@objc func playbackStalledHandler() {
delegate?.playerItemPlaybackStalled?(self)
}
// MARK: -
override init(asset _: AVAsset, automaticallyLoadedAssetKeys _: [String]?) {
fatalError("not implemented")
}
deinit {
NotificationCenter.default.removeObserver(self)
removeObserver(self, forKeyPath: "status")
resourceLoaderDelegate.session?.invalidateAndCancel()
}
}