diff --git a/apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift b/apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift index c3deec02a..89b7aee19 100644 --- a/apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/WelcomeView.swift @@ -33,11 +33,7 @@ struct WelcomeView: View { var headlineText: some View { Group { - if horizontalSizeClass == .compact { - Text("Everything you read. Safe, organized, and easy to share.") - } else { - Text("Everything you read. Safe,\norganized, and easy to share.") - } + Text("Never miss a great read.") } .font(.appLargeTitle) } diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift index 9c87efa6c..0d85d6462 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift @@ -36,7 +36,7 @@ enum DownloadPriority: String { } // Our observable object class -public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { +public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate, CachingPlayerItemDelegate { @Published public var state: AudioSessionState = .stopped @Published public var item: LinkedItem? @@ -49,7 +49,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { let networker: Networker var timer: Timer? - var player: AVAudioPlayer? + var player: AVPlayer? var downloadTask: Task? public init(appEnvironment: AppEnvironment, networker: Networker) { @@ -65,7 +65,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { } public func stop() { - player?.stop() + // player?.stop() clearNowPlayingInfo() timer = nil player = nil @@ -134,11 +134,16 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { case .scrubStarted: return case let .scrubEnded(seekTime): - player?.currentTime = seekTime + seek(to: seekTime) +// player?.currentTime = seekTime } } } + public func seek(to _: TimeInterval) { + // seek + } + public var currentVoice: String { "en-US-JennyNeural" } @@ -151,16 +156,19 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { state == .playing && self.item == item } - public func skipForward(seconds: Double) { - if let current = player?.currentTime { - player?.currentTime = min(duration, current + seconds) - } + public func skipForward(seconds _: Double) { +// if let current = player?.currentTime { +// seek(to: current + seconds) + //// player?.currentTime = min(duration, current + seconds) +// } } - public func skipBackwards(seconds: Double) { - if let current = player?.currentTime { - player?.currentTime = max(0, current - seconds) - } + public func skipBackwards(seconds _: Double) { +// if let current = player?.currentTime { +// seek(to: current + seconds) +// + //// player?.currentTime = max(0, current - seconds) +// } } public func fileNameForAudioFile(_ pageId: String) -> String { @@ -180,21 +188,21 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { let pageId = item!.unwrappedID downloadTask = Task { - let result = try? await downloadAudioFile(pageId: pageId, type: .mp3, priority: .high) - if Task.isCancelled { return } - - if result == nil { - DispatchQueue.main.async { - NSNotification.operationSuccess(message: "Error generating audio.") - self.stop() - } - } - - if let result = result, result.pending { - DispatchQueue.main.async { - NSNotification.operationSuccess(message: "Your audio is being generated.") - } - } +// let result = try? await downloadAudioFile(pageId: pageId, type: .mp3, priority: .high) +// if Task.isCancelled { return } +// +// if result == nil { +// DispatchQueue.main.async { +// NSNotification.operationSuccess(message: "Error generating audio.") +// self.stop() +// } +// } +// +// if let result = result, result.pending { +// DispatchQueue.main.async { +// NSNotification.operationSuccess(message: "Your audio is being generated.") +// } +// } DispatchQueue.main.async { self.startDownloadedAudioFile(pageId: pageId) @@ -208,28 +216,44 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { state = .stopped return } - - // TODO: Maybe check if app is active so it doesn't end up playing later? - - let audioUrl = pathForAudioFile(pageId: pageId) - if !FileManager.default.fileExists(atPath: audioUrl.path) { - stop() - return - } +// +// // TODO: Maybe check if app is active so it doesn't end up playing later? +// +// let audioUrl = pathForAudioFile(pageId: pageId) +// if !FileManager.default.fileExists(atPath: audioUrl.path) { +// stop() +// return +// } do { - try AVAudioSession.sharedInstance().setCategory(.playback) + try AVAudioSession.sharedInstance().setCategory(.playback, mode: .default, options: []) - player = try AVAudioPlayer(contentsOf: audioUrl) - player?.delegate = self - if player?.play() ?? false { - state = .playing - startTimer() - setupRemoteControl() - } + let token = ValetKey.authToken.value()! + let url = URL(string: "https://text-to-speech-streaming-bryle2uxwq-wl.a.run.app/?token=\(token)&q=1")! + print("LOADING URL: ", url) + + // let url = URL(string: "https://storage.googleapis.com/omnivore-demo-files/speech/062bfcc2-8d59-4880-8a67-fe6ee739c510.mp3")! +// let url = URL(string: "http://devimages.apple.com/iphone/samples/bipbop/bipbopall.m3u8")! + let playerItem = CachingPlayerItem(url: url) + playerItem.delegate = self + + player = AVPlayer(playerItem: playerItem) + print("created player: ", player, player?.error) + player?.automaticallyWaitsToMinimizeStalling = false + player?.play() + print("starting playing: ", player, player?.error) + +// +// player = try AVAudioPlayer(contentsOf: audioUrl) +// player?.delegate = self +// if player?.play() ?? false { + state = .playing + startTimer() + // setupRemoteControl() + // } } catch { print("error playing MP3 file", error) - try? FileManager.default.removeItem(atPath: audioUrl.path) + // try? FileManager.default.removeItem(atPath: audioUrl.path) state = .stopped } } @@ -259,7 +283,7 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { func startTimer() { if timer == nil { // Update every 100ms - timer = Timer.scheduledTimer(timeInterval: 0.1, target: self, selector: #selector(update(_:)), userInfo: nil, repeats: true) + timer = Timer.scheduledTimer(timeInterval: 10, target: self, selector: #selector(update(_:)), userInfo: nil, repeats: true) timer?.fire() } } @@ -278,26 +302,37 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { // Every second, get the current playing time of the player and refresh the status of the player progressslider @objc func update(_: Timer) { - if let player = player, player.isPlaying { - duration = player.duration - durationString = formatTimeInterval(duration) - - switch scrubState { - case .reset: - timeElapsed = player.currentTime - timeElapsedString = formatTimeInterval(timeElapsed) - if var nowPlaying = MPNowPlayingInfoCenter.default().nowPlayingInfo { - nowPlaying[MPMediaItemPropertyPlaybackDuration] = NSNumber(value: duration) - nowPlaying[MPNowPlayingInfoPropertyElapsedPlaybackTime] = NSNumber(value: timeElapsed) - MPNowPlayingInfoCenter.default().nowPlayingInfo = nowPlaying - } - case .scrubStarted: - break - case let .scrubEnded(seekTime): - scrubState = .reset - timeElapsed = seekTime - } + if let player = player { + print("current error: ", player.error) + print("current state", player.rate) + print("current position", player.currentTime()) + print("timeControlStatus", player.timeControlStatus.rawValue) + print("waiting Reason: ", player.reasonForWaitingToPlay?.rawValue) + print("currentItem.duration: ", player.currentItem?.duration) + print("status:", player.currentItem?.status.rawValue) + print("error:", player.currentItem?.error) + print("error log:", player.currentItem?.errorLog()) } +// if let player = player, player.isPlaying { +// duration = player.duration +// durationString = formatTimeInterval(duration) +// +// switch scrubState { +// case .reset: +// timeElapsed = player.currentTime +// timeElapsedString = formatTimeInterval(timeElapsed) +// if var nowPlaying = MPNowPlayingInfoCenter.default().nowPlayingInfo { +// nowPlaying[MPMediaItemPropertyPlaybackDuration] = NSNumber(value: duration) +// nowPlaying[MPNowPlayingInfoPropertyElapsedPlaybackTime] = NSNumber(value: timeElapsed) +// MPNowPlayingInfoCenter.default().nowPlayingInfo = nowPlaying +// } +// case .scrubStarted: +// break +// case let .scrubEnded(seekTime): +// scrubState = .reset +// timeElapsed = seekTime +// } +// } } func clearNowPlayingInfo() { @@ -353,7 +388,8 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { commandCenter.changePlaybackPositionCommand.isEnabled = true commandCenter.changePlaybackPositionCommand.addTarget { event -> MPRemoteCommandHandlerStatus in if let event = event as? MPChangePlaybackPositionCommandEvent { - self.player?.currentTime = event.positionTime + self.seek(to: event.positionTime) +// self.player?.currentTime = event.positionTime return .success } return .commandFailed @@ -456,4 +492,31 @@ public class AudioSession: NSObject, ObservableObject, AVAudioPlayerDelegate { default: () } } + + /// Is called when the media file is fully downloaded. + @objc func playerItem(_: CachingPlayerItem, didFinishDownloadingData data: Data) { + print("didFinishDownloadingData: ", data.underestimatedCount) + } + + /// Is called every time a new portion of data is received. + @objc func playerItem(_: CachingPlayerItem, didDownloadBytesSoFar bytesDownloaded: Int, outOf _: Int) { + print("didDownloadBytesSoFar: ", bytesDownloaded) + } + + /// Is called after initial prebuffering is finished, means + /// we are ready to play. + @objc func playerItemReadyToPlay(_: CachingPlayerItem) { + print("playerItemReadyToPlay") + } + + /// Is called when the data being downloaded did not arrive in time to + /// continue playback. + @objc func playerItemPlaybackStalled(_: CachingPlayerItem) { + print("playerItemPlaybackStalled") + } + + /// Is called on downloading error. + @objc func playerItem(_: CachingPlayerItem, downloadingFailedWith error: Error) { + print("downloadingFailedWith errpr", error) + } } diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/CachingPlayerItem.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/CachingPlayerItem.swift new file mode 100644 index 000000000..56b6e4252 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/CachingPlayerItem.swift @@ -0,0 +1,254 @@ +// From: https://github.com/neekeetab/CachingPlayerItem + +import AVFoundation +import Foundation + +private extension URL { + func withScheme(_ scheme: String) -> URL? { + var components = URLComponents(url: self, resolvingAgainstBaseURL: false) + components?.scheme = scheme + return components?.url + } +} + +@objc protocol CachingPlayerItemDelegate { + /// Is called when the media file is fully downloaded. + @objc optional func playerItem(_ playerItem: CachingPlayerItem, didFinishDownloadingData data: Data) + + /// Is called every time a new portion of data is received. + @objc optional func playerItem(_ playerItem: CachingPlayerItem, didDownloadBytesSoFar bytesDownloaded: Int, outOf bytesExpected: Int) + + /// Is called after initial prebuffering is finished, means + /// we are ready to play. + @objc optional func playerItemReadyToPlay(_ playerItem: CachingPlayerItem) + + /// Is called when the data being downloaded did not arrive in time to + /// continue playback. + @objc optional func playerItemPlaybackStalled(_ playerItem: CachingPlayerItem) + + /// Is called on downloading error. + @objc optional func playerItem(_ playerItem: CachingPlayerItem, downloadingFailedWith error: Error) +} + +open class CachingPlayerItem: AVPlayerItem { + class ResourceLoaderDelegate: NSObject, AVAssetResourceLoaderDelegate, URLSessionDelegate, URLSessionDataDelegate, URLSessionTaskDelegate { + var playingFromData = false + var mimeType: String? // is required when playing from Data + var session: URLSession? + var mediaData: Data? + var response: URLResponse? + var pendingRequests = Set() + weak var owner: CachingPlayerItem? + + func resourceLoader(_: AVAssetResourceLoader, shouldWaitForLoadingOfRequestedResource loadingRequest: AVAssetResourceLoadingRequest) -> Bool { + if playingFromData { + // Nothing to load. + + } else if session == nil { + // If we're playing from a url, we need to download the file. + // We start loading the file on first request only. + guard let initialUrl = owner?.url else { + fatalError("internal inconsistency") + } + + startDataRequest(with: initialUrl) + } + + pendingRequests.insert(loadingRequest) + processPendingRequests() + return true + } + + func startDataRequest(with url: URL) { + let configuration = URLSessionConfiguration.default + configuration.requestCachePolicy = .reloadIgnoringLocalAndRemoteCacheData + session = URLSession(configuration: configuration, delegate: self, delegateQueue: nil) + session?.dataTask(with: url).resume() + } + + func resourceLoader(_: AVAssetResourceLoader, didCancel loadingRequest: AVAssetResourceLoadingRequest) { + pendingRequests.remove(loadingRequest) + } + + // MARK: URLSession delegate + + func urlSession(_: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { + mediaData?.append(data) + processPendingRequests() + owner?.delegate?.playerItem?(owner!, didDownloadBytesSoFar: mediaData!.count, outOf: Int(dataTask.countOfBytesExpectedToReceive)) + } + + func urlSession(_: URLSession, dataTask _: URLSessionDataTask, didReceive response: URLResponse, completionHandler: @escaping (URLSession.ResponseDisposition) -> Void) { + completionHandler(Foundation.URLSession.ResponseDisposition.allow) + mediaData = Data() + self.response = response + processPendingRequests() + } + + func urlSession(_: URLSession, task _: URLSessionTask, didCompleteWithError error: Error?) { + if let errorUnwrapped = error { + owner?.delegate?.playerItem?(owner!, downloadingFailedWith: errorUnwrapped) + return + } + processPendingRequests() + owner?.delegate?.playerItem?(owner!, didFinishDownloadingData: mediaData!) + } + + // MARK: - + + func processPendingRequests() { + // get all fullfilled requests + let requestsFulfilled = Set(pendingRequests.compactMap { + self.fillInContentInformationRequest($0.contentInformationRequest) + if self.haveEnoughDataToFulfillRequest($0.dataRequest!) { + $0.finishLoading() + return $0 + } + return nil + }) + + // remove fulfilled requests from pending requests + _ = requestsFulfilled.map { self.pendingRequests.remove($0) } + } + + func fillInContentInformationRequest(_ contentInformationRequest: AVAssetResourceLoadingContentInformationRequest?) { + // if we play from Data we make no url requests, therefore we have no responses, so we need to fill in contentInformationRequest manually + if playingFromData { + contentInformationRequest?.contentType = mimeType + contentInformationRequest?.contentLength = Int64(mediaData!.count) + contentInformationRequest?.isByteRangeAccessSupported = true + return + } + + guard let responseUnwrapped = response else { + // have no response from the server yet + return + } + + contentInformationRequest?.contentType = responseUnwrapped.mimeType + contentInformationRequest?.contentLength = responseUnwrapped.expectedContentLength + contentInformationRequest?.isByteRangeAccessSupported = true + } + + func haveEnoughDataToFulfillRequest(_ dataRequest: AVAssetResourceLoadingDataRequest) -> Bool { + let requestedOffset = Int(dataRequest.requestedOffset) + let requestedLength = dataRequest.requestedLength + let currentOffset = Int(dataRequest.currentOffset) + + guard let songDataUnwrapped = mediaData, + songDataUnwrapped.count > currentOffset + else { + // Don't have any data at all for this request. + return false + } + + let bytesToRespond = min(songDataUnwrapped.count - currentOffset, requestedLength) + let dataToRespond = songDataUnwrapped.subdata(in: Range(uncheckedBounds: (currentOffset, currentOffset + bytesToRespond))) + dataRequest.respond(with: dataToRespond) + + return songDataUnwrapped.count >= requestedLength + requestedOffset + } + + deinit { + session?.invalidateAndCancel() + } + } + + fileprivate let resourceLoaderDelegate = ResourceLoaderDelegate() + fileprivate let url: URL + fileprivate let initialScheme: String? + fileprivate var customFileExtension: String? + + weak var delegate: CachingPlayerItemDelegate? + + open func download() { + if resourceLoaderDelegate.session == nil { + resourceLoaderDelegate.startDataRequest(with: url) + } + } + + private let cachingPlayerItemScheme = "cachingPlayerItemScheme" + + /// Is used for playing remote files. + convenience init(url: URL) { + self.init(url: url, customFileExtension: nil) + } + + /// Override/append custom file extension to URL path. + /// This is required for the player to work correctly with the intended file type. + init(url: URL, customFileExtension _: String?) { + guard let components = URLComponents(url: url, resolvingAgainstBaseURL: false), + let scheme = components.scheme, + var urlWithCustomScheme = url.withScheme(cachingPlayerItemScheme) + else { + fatalError("Urls without a scheme are not supported") + } + + self.url = url + self.initialScheme = scheme + + if let ext = customFileExtension { + urlWithCustomScheme.deletePathExtension() + urlWithCustomScheme.appendPathExtension(ext) + self.customFileExtension = ext + } + + let mimeType = "audio/mpeg" + let asset = AVURLAsset(url: url, options: ["AVURLAssetOutOfBandMIMETypeKey": mimeType]) + asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main) + super.init(asset: asset, automaticallyLoadedAssetKeys: nil) + + resourceLoaderDelegate.owner = self + + addObserver(self, forKeyPath: "status", options: NSKeyValueObservingOptions.new, context: nil) + + NotificationCenter.default.addObserver(self, selector: #selector(playbackStalledHandler), name: NSNotification.Name.AVPlayerItemPlaybackStalled, object: self) + } + + /// Is used for playing from Data. + init(data: Data, mimeType: String, fileExtension: String) { + guard let fakeUrl = URL(string: cachingPlayerItemScheme + "://whatever/file.\(fileExtension)") else { + fatalError("internal inconsistency") + } + + self.url = fakeUrl + self.initialScheme = nil + + resourceLoaderDelegate.mediaData = data + resourceLoaderDelegate.playingFromData = true + resourceLoaderDelegate.mimeType = mimeType + + let asset = AVURLAsset(url: fakeUrl) + asset.resourceLoader.setDelegate(resourceLoaderDelegate, queue: DispatchQueue.main) + super.init(asset: asset, automaticallyLoadedAssetKeys: nil) + resourceLoaderDelegate.owner = self + + addObserver(self, forKeyPath: "status", options: NSKeyValueObservingOptions.new, context: nil) + + NotificationCenter.default.addObserver(self, selector: #selector(playbackStalledHandler), name: NSNotification.Name.AVPlayerItemPlaybackStalled, object: self) + } + + // MARK: KVO + + override open func observeValue(forKeyPath _: String?, of _: Any?, change _: [NSKeyValueChangeKey: Any]?, context _: UnsafeMutableRawPointer?) { + delegate?.playerItemReadyToPlay?(self) + } + + // MARK: Notification hanlers + + @objc func playbackStalledHandler() { + delegate?.playerItemPlaybackStalled?(self) + } + + // MARK: - + + override init(asset _: AVAsset, automaticallyLoadedAssetKeys _: [String]?) { + fatalError("not implemented") + } + + deinit { + NotificationCenter.default.removeObserver(self) + removeObserver(self, forKeyPath: "status") + resourceLoaderDelegate.session?.invalidateAndCancel() + } +}