diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift index 2879b5366..fb669c83b 100644 --- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift +++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift @@ -67,7 +67,7 @@ public struct MiniPlayer: View { }, label: { Image(systemName: playPauseButtonImage) - .font(expanded ? .system(size: 64.0, weight: .thin) : .appTitleTwo) + .font(expanded ? .system(size: 56.0, weight: .thin) : .appTitleTwo) } )) } @@ -145,16 +145,16 @@ public struct MiniPlayer: View { // swiftlint:disable:next function_body_length func playerContent(_ itemAudioProperties: LinkedItemAudioProperties) -> some View { GeometryReader { geom in - VStack { + VStack(spacing: 0) { if expanded { ZStack { closeButton .padding(.top, 24) .frame(maxWidth: .infinity, alignment: .leading) -// shareButton -// .padding(.top, 8) -// .frame(maxWidth: .infinity, alignment: .trailing) + // shareButton + // .padding(.top, 8) + // .frame(maxWidth: .infinity, alignment: .trailing) Capsule() .fill(.gray) @@ -162,64 +162,115 @@ public struct MiniPlayer: View { .padding(.top, 8) .transition(.opacity) } - } + } else { + HStack(alignment: .center) { + let maxSize = 2 * (min(geom.size.width, geom.size.height) / 3) + let dim = 64.0 // expanded ? maxSize : 64 - Spacer(minLength: 0) + if let imageURL = itemAudioProperties.imageURL { + AsyncImage(url: imageURL) { phase in + if let image = phase.image { + image + .resizable() + .aspectRatio(contentMode: .fill) + .frame(width: dim, height: dim) + .cornerRadius(6) + } else if phase.error != nil { + defaultArtwork(forDimensions: dim) + } else { + Color.appButtonBackground + .frame(width: dim, height: dim) + .cornerRadius(6) + } + } + } else { + defaultArtwork(forDimensions: dim) + } - HStack { - let maxSize = 2 * (min(geom.size.width, geom.size.height) / 3) - let dim = expanded ? maxSize : 64 + // if !expanded { + VStack { + Text(itemAudioProperties.title) + .font(.appCallout) + .foregroundColor(.appGrayTextContrast) + .fixedSize(horizontal: false, vertical: false) + .frame(maxWidth: .infinity, alignment: .leading) + .matchedGeometryEffect(id: "ArticleTitle", in: animation) - if let imageURL = itemAudioProperties.imageURL { - AsyncImage(url: imageURL) { phase in - if let image = phase.image { - image - .resizable() - .aspectRatio(contentMode: .fill) - .frame(width: dim, height: dim) - .cornerRadius(6) - } else if phase.error != nil { - defaultArtwork(forDimensions: dim) - } else { - Color.appButtonBackground - .frame(width: dim, height: dim) - .cornerRadius(6) + if let byline = itemAudioProperties.byline { + Text(byline) + .font(.appCaption) + .lineSpacing(1.25) + .foregroundColor(.appGrayText) + .fixedSize(horizontal: false, vertical: false) + .frame(maxWidth: .infinity, alignment: .leading) + // .matchedGeometryEffect(id: "ArticleTitle", in: animation) } } - } else { - defaultArtwork(forDimensions: dim) - } - - if !expanded { - Text(itemAudioProperties.title) - .font(.appCallout) - .lineSpacing(1.25) - .foregroundColor(.appGrayTextContrast) - .fixedSize(horizontal: false, vertical: false) - .frame(maxWidth: .infinity, alignment: .leading) - .matchedGeometryEffect(id: "ArticleTitle", in: animation) playPauseButtonItem .frame(width: 28, height: 28) stopButton .frame(width: 28, height: 28) - } + }.frame(maxHeight: .infinity) } - Spacer() + // Spacer(minLength: 0) if expanded { - Marquee(text: itemAudioProperties.title, font: UIFont(name: "Inter-Regular", size: 22)!) - .foregroundColor(.appGrayTextContrast) - .onTapGesture { - viewArticle() - } +// Marquee(text: itemAudioProperties.title, font: UIFont(name: "Inter-Regular", size: 22)!) +// .foregroundColor(.appGrayTextContrast) +// .onTapGesture { +// viewArticle() +// } +// +// if let byline = itemAudioProperties.byline { +// Marquee(text: byline, font: UIFont(name: "Inter-Regular", size: 16)!) +// .foregroundColor(.appGrayText) +// } - if let byline = itemAudioProperties.byline { - Marquee(text: byline, font: UIFont(name: "Inter-Regular", size: 16)!) - .foregroundColor(.appGrayText) + Group { + Text(audioController.readText) + .font(.textToSpeechRead.leading(.loose)) + .foregroundColor(Color.appGrayTextContrast) + + + Text(audioController.unreadText) + .font(.textToSpeechRead.leading(.loose)) + .foregroundColor(Color.appGrayText) } + .padding(24) + +// ScrollView { +// ScrollViewReader { _ in +// ForEach(Array(self.audioController.readText.enumerated()), id: \.1.self) { index, text in +// Text(text) +// .font(.textToSpeechRead) +// .lineSpacing(2.5) +// .padding() +// .id(index) +// } +// +// ForEach(Array(self.audioController.unreadText.enumerated()), id: \.1.self) { index, text in +// Text(text) +// .font(.textToSpeechUnread) +// .opacity(0.55) +// .lineSpacing(2.5) +// .padding() +// .id(index) +// +// }.onChange(of: self.audioController.currentAudioIndex) { _ in +// // withAnimation(.spring()) { +// // proxy.scrollTo(value, anchor: .top) +// // } +// } +// } +// + //// Text("This is where the main content would go") + + //// Text("its multiple lines of text.") + + //// Text("It would probably need to scroll as our text segments are pretty big") +// } + + Spacer() ScrubberView(value: $audioController.timeElapsed, minValue: 0, maxValue: self.audioController.duration, @@ -241,7 +292,7 @@ public struct MiniPlayer: View { .foregroundColor(.appGrayText) } - HStack { + HStack(alignment: .center, spacing: 36) { Menu { playbackRateButton(rate: 1.0, title: "1.0×", selected: audioController.playbackRate == 1.0) playbackRateButton(rate: 1.1, title: "1.1×", selected: audioController.playbackRate == 1.1) @@ -268,8 +319,7 @@ public struct MiniPlayer: View { ) playPauseButtonItem - .frame(width: 64, height: 64) - .padding(32) + .frame(width: 56, height: 56) Button( action: { self.audioController.skipForward(seconds: 30) }, @@ -291,10 +341,10 @@ public struct MiniPlayer: View { .contentShape(Rectangle()) } .padding(8) - } + }.padding(.bottom, 16) } } - .padding(EdgeInsets(top: 0, leading: expanded ? 24 : 6, bottom: 0, trailing: expanded ? 24 : 6)) + .padding(EdgeInsets(top: 0, leading: 6, bottom: 0, trailing: 6)) .background( Color.systemBackground .shadow(color: expanded ? .clear : .gray.opacity(0.33), radius: 8, x: 0, y: 4) diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift index f34b5e935..e2a10681e 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift @@ -37,6 +37,8 @@ class SpeechPlayerItem: AVPlayerItem { let resourceLoaderDelegate = ResourceLoaderDelegate() let session: AudioController let speechItem: SpeechItem + var speechMarks: [SpeechMark]? + let completed: () -> Void var observer: Any? @@ -123,12 +125,16 @@ class SpeechPlayerItem: AVPlayerItem { } // TODO: how do we want to propogate this and handle it in the player - let audioData = try? await SpeechSynthesizer.download(speechItem: speechItem, session: self.session) + let speechData = try? await SpeechSynthesizer.download(speechItem: speechItem, session: self.session) DispatchQueue.main.async { - if audioData == nil { + if speechData == nil { self.session = nil } - self.mediaData = audioData + if let owner = self.owner, let speechData = speechData { + owner.speechMarks = speechData.speechMarks + } + self.mediaData = speechData?.audioData + self.processPendingRequests() } } @@ -189,6 +195,10 @@ class SpeechPlayerItem: AVPlayerItem { // swiftlint:disable all public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate { @Published public var state: AudioControllerState = .stopped + @Published public var currentAudioIndex: Int = 0 + @Published public var readText: String = "" + @Published public var unreadText: String = "" + @Published public var itemAudioProperties: LinkedItemAudioProperties? @Published public var timeElapsed: TimeInterval = 0 @@ -202,6 +212,7 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate var timer: Timer? var player: AVQueuePlayer? + var observer: Any? var document: SpeechDocument? var synthesizer: SpeechSynthesizer? var durations: [Double]? @@ -214,6 +225,11 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate self.voiceList = generateVoiceList() } + deinit { + player = nil + observer = nil + } + public func play(itemAudioProperties: LinkedItemAudioProperties) { stop() @@ -237,8 +253,10 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate player?.replaceCurrentItem(with: nil) player?.removeAllItems() + document = nil timer = nil player = nil + observer = nil synthesizer = nil itemAudioProperties = nil @@ -449,6 +467,80 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate return Voices.Pairs.first(where: { $0.firstKey == voice || $0.secondKey == voice }) } + struct TextNode: Codable { + let to: String + let from: String + let heading: String + let body: String + } + + public var textItems: [String]? { + if let document = self.document { + return document.utterances.map(\.text) + } + return nil + } + + func updateReadText() { + if let textItems = textItems, let item = player?.currentItem as? SpeechPlayerItem, let speechMarks = item.speechMarks { + // up till: + var currentItemOffset = 0 + for i in 0 ..< speechMarks.count { + if speechMarks[i].time ?? 0 < 0 { + continue + } + if (speechMarks[i].time ?? 0.0) > CMTimeGetSeconds(item.currentTime()) * 1000 { + currentItemOffset = speechMarks[i].start ?? 0 + break + } + } + // check to see if we are greater than all + if let last = speechMarks.last, let lastTime = last.time { + if CMTimeGetSeconds(item.currentTime()) * 1000 > lastTime { + currentItemOffset = (last.start ?? 0) + (last.length ?? 0) + } + } + + let idx = item.speechItem.audioIdx + let currentItem = textItems[idx] + let currentReadIndex = currentItem.index(currentItem.startIndex, offsetBy: currentItemOffset) + let lastItem = String(currentItem[..]*>", options: .caseInsensitive) { +// let modString = regex.stringByReplacingMatches(in: text, options: [], range: NSRange(location: 0, length: text.count), withTemplate: "") +// return modString +// } +// return "" +// }) + } else { + readText = "" + } + } + + func updateUnreadText() { +// if let textItems = textItems, let item = player?.currentItem as? SpeechPlayerItem { +// let idx = item.speechItem.audioIdx +// +// +// unreadText = Array(textItems[idx...].map { text in +// if let regex = try? NSRegularExpression(pattern: "<[^>]*>", options: .caseInsensitive) { +// let modString = regex.stringByReplacingMatches(in: text, options: [], range: NSRange(location: 0, length: text.count), withTemplate: "") +// return modString +// } +// return "" +// }) +// } else { +// unreadText = [] +// } + } + public func getPreferredVoice(forLanguage language: String) -> String { UserDefaults.standard.string(forKey: "\(language)-\(UserDefaultKey.textToSpeechPreferredVoice.rawValue)") ?? currentVoiceLanguage.defaultVoice } @@ -593,6 +685,13 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate } player = AVQueuePlayer(items: []) + if let player = player { + observer = player.observe(\.currentItem, options: [.new]) { _, _ in + print("current item did change: ", (player.currentItem as? SpeechPlayerItem)?.speechItem.audioIdx) + self.currentAudioIndex = (player.currentItem as? SpeechPlayerItem)?.speechItem.audioIdx ?? 0 + } + } + let synthesizer = SpeechSynthesizer(appEnvironment: appEnvironment, networker: networker, document: document) durations = synthesizer.estimatedDurations(forSpeed: playbackRate) self.synthesizer = synthesizer @@ -660,7 +759,7 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate func startTimer() { if timer == nil { - timer = Timer.scheduledTimer(timeInterval: 0.1, target: self, selector: #selector(fireTimer), userInfo: nil, repeats: true) + timer = Timer.scheduledTimer(timeInterval: 0.2, target: self, selector: #selector(fireTimer), userInfo: nil, repeats: true) timer?.fire() } } @@ -676,6 +775,8 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate duration = durations.reduce(0, +) durationString = formatTimeInterval(duration) } + + updateReadText() } if let player = player { diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift index 7686be306..6c67f5fc0 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift @@ -60,8 +60,19 @@ struct SpeechItem { let audioIdx: Int let urlRequest: URLRequest let localAudioURL: URL + let localSpeechURL: URL + let text: String } +struct SpeechMark: Decodable, Encodable { + let word: String? + let time: Double? + let start: Int? + let length: Int? +} + +struct SpeechData {} + struct SpeechSynthesizer { typealias Element = SpeechItem let document: SpeechDocument @@ -91,9 +102,15 @@ struct SpeechSynthesizer { let voiceStr = utterance.voice ?? document.defaultVoice let segmentStr = String(format: "%04d", arguments: [idx]) let localAudioURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3") + let localSpeechURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).speechMarks") if let request = urlRequestFor(utterance: utterance) { - let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: idx, urlRequest: request, localAudioURL: localAudioURL) + let item = SpeechItem(htmlIdx: utterance.idx, + audioIdx: idx, + urlRequest: request, + localAudioURL: localAudioURL, + localSpeechURL: localSpeechURL, + text: utterance.text) return item } @@ -108,9 +125,15 @@ struct SpeechSynthesizer { let voiceStr = utterance.voice ?? document.defaultVoice let segmentStr = String(format: "%04d", arguments: [idx]) let localAudioURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3") + let localSpeechURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3") if let request = urlRequestFor(utterance: utterance) { - let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: idx, urlRequest: request, localAudioURL: localAudioURL) + let item = SpeechItem(htmlIdx: utterance.idx, + audioIdx: idx, + urlRequest: request, + localAudioURL: localAudioURL, + localSpeechURL: localSpeechURL, + text: utterance.text) result.append(item) } else { // TODO: How do we want to handle completely skipped paragraphs? @@ -138,11 +161,17 @@ struct SpeechSynthesizer { static func download(speechItem: SpeechItem, redownloadCached: Bool = false, - session: URLSession? = URLSession.shared) async throws -> Data? + session: URLSession? = URLSession.shared) async throws -> SynthesizeData? { + let decoder = JSONDecoder() + if !redownloadCached, FileManager.default.fileExists(atPath: speechItem.localAudioURL.path) { - if let localData = try? Data(contentsOf: speechItem.localAudioURL) { - return localData + if let speechMarksData = try? Data(contentsOf: speechItem.localSpeechURL), + let speechMarks = try? decoder.decode([SpeechMark].self, from: speechMarksData), + let localData = try? Data(contentsOf: speechItem.localAudioURL) + { + print("CACHED DATA LENGTH: ", localData.count) + // return SynthesizeData(audioData: localData, speechMarks: speechMarks) } } @@ -161,20 +190,36 @@ struct SpeechSynthesizer { .urls(for: .cachesDirectory, in: .userDomainMask)[0] .appendingPathComponent(UUID().uuidString + ".mp3") + let tempSMPath = FileManager.default + .urls(for: .cachesDirectory, in: .userDomainMask)[0] + .appendingPathComponent(UUID().uuidString + ".speechMarks") + do { - let decoder = JSONDecoder() - let jsonData = try decoder.decode(SynthesizeResult.self, from: data) + print("SPEECH DATA: ", String(decoding: data, as: UTF8.self)) + + let jsonData = try decoder.decode(SynthesizeResult.self, from: data) as SynthesizeResult let audioData = Data(fromHexEncodedString: jsonData.audioData)! if audioData.count < 1 { throw BasicError.message(messageText: "Audio data is empty") } + print("AUDIO DATA LENGTH: ", audioData.count) + try audioData.write(to: tempPath) try? FileManager.default.removeItem(at: speechItem.localAudioURL) try FileManager.default.moveItem(at: tempPath, to: speechItem.localAudioURL) - return audioData + let encoder = JSONEncoder() + let speechMarksData = try encoder.encode(jsonData.speechMarks) + try speechMarksData.write(to: tempSMPath) + try? FileManager.default.removeItem(at: speechItem.localSpeechURL) + try FileManager.default.moveItem(at: tempSMPath, to: speechItem.localSpeechURL) + + print("DOWNLOADED SPEECH MARKS: ", jsonData.speechMarks) + + return SynthesizeData(audioData: audioData, speechMarks: jsonData.speechMarks) } catch { + print("ERROR WRITING DATA", error) let errorMessage = "audioFetch failed. could not write MP3 data to disk" throw BasicError.message(messageText: errorMessage) } @@ -183,7 +228,12 @@ struct SpeechSynthesizer { struct SynthesizeResult: Decodable { let audioData: String -// let speechMarks: Any? + let speechMarks: [SpeechMark] +} + +struct SynthesizeData: Decodable { + let audioData: Data + let speechMarks: [SpeechMark] } extension Data { diff --git a/apple/OmnivoreKit/Sources/Views/Fonts.swift b/apple/OmnivoreKit/Sources/Views/Fonts.swift index 40385127c..8e5f70e73 100644 --- a/apple/OmnivoreKit/Sources/Views/Fonts.swift +++ b/apple/OmnivoreKit/Sources/Views/Fonts.swift @@ -16,6 +16,14 @@ public extension Font { /// 22pt, Inter-Regular static var appTitleTwo: Font { + Font.custom(InterFont.regular.rawValue, size: 28, relativeTo: .title2) + } + + static var textToSpeechRead: Font { + Font.custom(InterFont.bold.rawValue, size: 28, relativeTo: .title2) + } + + static var textToSpeechUnread: Font { Font.custom(InterFont.regular.rawValue, size: 22, relativeTo: .title2) }