prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

WIP: follow along text for tts

Browse source
This commit is contained in:
Jackson Harper 2022-10-04 11:46:50 +08:00
parent 30a0c67f8c
commit f9e2809a10
4 changed files with 273 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

152
apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift
View file

@ -67,7 +67,7 @@ public struct MiniPlayer: View {
},
label: {
Image(systemName: playPauseButtonImage)
.font(expanded ? .system(size: 64.0, weight: .thin) : .appTitleTwo)
.font(expanded ? .system(size: 56.0, weight: .thin) : .appTitleTwo)
}
))
}
@ -145,16 +145,16 @@ public struct MiniPlayer: View {
// swiftlint:disable:next function_body_length
func playerContent(_ itemAudioProperties: LinkedItemAudioProperties) -> some View {
GeometryReader { geom in
VStack {
VStack(spacing: 0) {
if expanded {
ZStack {
closeButton
.padding(.top, 24)
.frame(maxWidth: .infinity, alignment: .leading)
// shareButton
// .padding(.top, 8)
// .frame(maxWidth: .infinity, alignment: .trailing)
// shareButton
// .padding(.top, 8)
// .frame(maxWidth: .infinity, alignment: .trailing)
Capsule()
.fill(.gray)
@ -162,64 +162,115 @@ public struct MiniPlayer: View {
.padding(.top, 8)
.transition(.opacity)
}
}
} else {
HStack(alignment: .center) {
let maxSize = 2 * (min(geom.size.width, geom.size.height) / 3)
let dim = 64.0 // expanded ? maxSize : 64
Spacer(minLength: 0)
if let imageURL = itemAudioProperties.imageURL {
AsyncImage(url: imageURL) { phase in
if let image = phase.image {
image
.resizable()
.aspectRatio(contentMode: .fill)
.frame(width: dim, height: dim)
.cornerRadius(6)
} else if phase.error != nil {
defaultArtwork(forDimensions: dim)
} else {
Color.appButtonBackground
.frame(width: dim, height: dim)
.cornerRadius(6)
}
}
} else {
defaultArtwork(forDimensions: dim)
}
HStack {
let maxSize = 2 * (min(geom.size.width, geom.size.height) / 3)
let dim = expanded ? maxSize : 64
// if !expanded {
VStack {
Text(itemAudioProperties.title)
.font(.appCallout)
.foregroundColor(.appGrayTextContrast)
.fixedSize(horizontal: false, vertical: false)
.frame(maxWidth: .infinity, alignment: .leading)
.matchedGeometryEffect(id: "ArticleTitle", in: animation)
if let imageURL = itemAudioProperties.imageURL {
AsyncImage(url: imageURL) { phase in
if let image = phase.image {
image
.resizable()
.aspectRatio(contentMode: .fill)
.frame(width: dim, height: dim)
.cornerRadius(6)
} else if phase.error != nil {
defaultArtwork(forDimensions: dim)
} else {
Color.appButtonBackground
.frame(width: dim, height: dim)
.cornerRadius(6)
if let byline = itemAudioProperties.byline {
Text(byline)
.font(.appCaption)
.lineSpacing(1.25)
.foregroundColor(.appGrayText)
.fixedSize(horizontal: false, vertical: false)
.frame(maxWidth: .infinity, alignment: .leading)
// .matchedGeometryEffect(id: "ArticleTitle", in: animation)
}
}
} else {
defaultArtwork(forDimensions: dim)
}
if !expanded {
Text(itemAudioProperties.title)
.font(.appCallout)
.lineSpacing(1.25)
.foregroundColor(.appGrayTextContrast)
.fixedSize(horizontal: false, vertical: false)
.frame(maxWidth: .infinity, alignment: .leading)
.matchedGeometryEffect(id: "ArticleTitle", in: animation)
playPauseButtonItem
.frame(width: 28, height: 28)
stopButton
.frame(width: 28, height: 28)
}
}.frame(maxHeight: .infinity)
}
Spacer()
// Spacer(minLength: 0)
if expanded {
Marquee(text: itemAudioProperties.title, font: UIFont(name: "Inter-Regular", size: 22)!)
.foregroundColor(.appGrayTextContrast)
.onTapGesture {
viewArticle()
}
// Marquee(text: itemAudioProperties.title, font: UIFont(name: "Inter-Regular", size: 22)!)
// .foregroundColor(.appGrayTextContrast)
// .onTapGesture {
// viewArticle()
// }
//
// if let byline = itemAudioProperties.byline {
// Marquee(text: byline, font: UIFont(name: "Inter-Regular", size: 16)!)
// .foregroundColor(.appGrayText)
// }
if let byline = itemAudioProperties.byline {
Marquee(text: byline, font: UIFont(name: "Inter-Regular", size: 16)!)
.foregroundColor(.appGrayText)
Group {
Text(audioController.readText)
.font(.textToSpeechRead.leading(.loose))
.foregroundColor(Color.appGrayTextContrast)
+
Text(audioController.unreadText)
.font(.textToSpeechRead.leading(.loose))
.foregroundColor(Color.appGrayText)
}
.padding(24)
// ScrollView {
// ScrollViewReader { _ in
// ForEach(Array(self.audioController.readText.enumerated()), id: \.1.self) { index, text in
// Text(text)
// .font(.textToSpeechRead)
// .lineSpacing(2.5)
// .padding()
// .id(index)
// }
//
// ForEach(Array(self.audioController.unreadText.enumerated()), id: \.1.self) { index, text in
// Text(text)
// .font(.textToSpeechUnread)
// .opacity(0.55)
// .lineSpacing(2.5)
// .padding()
// .id(index)
//
// }.onChange(of: self.audioController.currentAudioIndex) { _ in
// // withAnimation(.spring()) {
// // proxy.scrollTo(value, anchor: .top)
// // }
// }
// }
//
//// Text("This is where the main content would go") +
//// Text("its multiple lines of text.") +
//// Text("It would probably need to scroll as our text segments are pretty big")
// }
Spacer()
ScrubberView(value: $audioController.timeElapsed,
minValue: 0, maxValue: self.audioController.duration,
@ -241,7 +292,7 @@ public struct MiniPlayer: View {
.foregroundColor(.appGrayText)
}
HStack {
HStack(alignment: .center, spacing: 36) {
Menu {
playbackRateButton(rate: 1.0, title: "1.0×", selected: audioController.playbackRate == 1.0)
playbackRateButton(rate: 1.1, title: "1.1×", selected: audioController.playbackRate == 1.1)
@ -268,8 +319,7 @@ public struct MiniPlayer: View {
)
playPauseButtonItem
.frame(width: 64, height: 64)
.padding(32)
.frame(width: 56, height: 56)
Button(
action: { self.audioController.skipForward(seconds: 30) },
@ -291,10 +341,10 @@ public struct MiniPlayer: View {
.contentShape(Rectangle())
}
.padding(8)
}
}.padding(.bottom, 16)
}
}
.padding(EdgeInsets(top: 0, leading: expanded ? 24 : 6, bottom: 0, trailing: expanded ? 24 : 6))
.padding(EdgeInsets(top: 0, leading: 6, bottom: 0, trailing: 6))
.background(
Color.systemBackground
.shadow(color: expanded ? .clear : .gray.opacity(0.33), radius: 8, x: 0, y: 4)

109
apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift
View file

@ -37,6 +37,8 @@ class SpeechPlayerItem: AVPlayerItem {
let resourceLoaderDelegate = ResourceLoaderDelegate()
let session: AudioController
let speechItem: SpeechItem
var speechMarks: [SpeechMark]?
let completed: () -> Void
var observer: Any?
@ -123,12 +125,16 @@ class SpeechPlayerItem: AVPlayerItem {
}
// TODO: how do we want to propogate this and handle it in the player
let audioData = try? await SpeechSynthesizer.download(speechItem: speechItem, session: self.session)
let speechData = try? await SpeechSynthesizer.download(speechItem: speechItem, session: self.session)
DispatchQueue.main.async {
if audioData == nil {
if speechData == nil {
self.session = nil
}
self.mediaData = audioData
if let owner = self.owner, let speechData = speechData {
owner.speechMarks = speechData.speechMarks
}
self.mediaData = speechData?.audioData
self.processPendingRequests()
}
}
@ -189,6 +195,10 @@ class SpeechPlayerItem: AVPlayerItem {
// swiftlint:disable all
public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate {
@Published public var state: AudioControllerState = .stopped
@Published public var currentAudioIndex: Int = 0
@Published public var readText: String = ""
@Published public var unreadText: String = ""
@Published public var itemAudioProperties: LinkedItemAudioProperties?
@Published public var timeElapsed: TimeInterval = 0
@ -202,6 +212,7 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
var timer: Timer?
var player: AVQueuePlayer?
var observer: Any?
var document: SpeechDocument?
var synthesizer: SpeechSynthesizer?
var durations: [Double]?
@ -214,6 +225,11 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
self.voiceList = generateVoiceList()
}
deinit {
player = nil
observer = nil
}
public func play(itemAudioProperties: LinkedItemAudioProperties) {
stop()
@ -237,8 +253,10 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
player?.replaceCurrentItem(with: nil)
player?.removeAllItems()
document = nil
timer = nil
player = nil
observer = nil
synthesizer = nil
itemAudioProperties = nil
@ -449,6 +467,80 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
return Voices.Pairs.first(where: { $0.firstKey == voice || $0.secondKey == voice })
}
struct TextNode: Codable {
let to: String
let from: String
let heading: String
let body: String
}
public var textItems: [String]? {
if let document = self.document {
return document.utterances.map(\.text)
}
return nil
}
func updateReadText() {
if let textItems = textItems, let item = player?.currentItem as? SpeechPlayerItem, let speechMarks = item.speechMarks {
// up till:
var currentItemOffset = 0
for i in 0 ..< speechMarks.count {
if speechMarks[i].time ?? 0 < 0 {
continue
}
if (speechMarks[i].time ?? 0.0) > CMTimeGetSeconds(item.currentTime()) * 1000 {
currentItemOffset = speechMarks[i].start ?? 0
break
}
}
// check to see if we are greater than all
if let last = speechMarks.last, let lastTime = last.time {
if CMTimeGetSeconds(item.currentTime()) * 1000 > lastTime {
currentItemOffset = (last.start ?? 0) + (last.length ?? 0)
}
}
let idx = item.speechItem.audioIdx
let currentItem = textItems[idx]
let currentReadIndex = currentItem.index(currentItem.startIndex, offsetBy: currentItemOffset)
let lastItem = String(currentItem[..<currentReadIndex])
let lastItemAfter = String(currentItem[currentReadIndex...])
// print("LAST ITEM: ", lastItem)
//
readText = lastItem
unreadText = lastItemAfter
// readText = Array((textItems[..<idx] + [lastItem]).map { text in
// if let regex = try? NSRegularExpression(pattern: "<[^>]*>", options: .caseInsensitive) {
// let modString = regex.stringByReplacingMatches(in: text, options: [], range: NSRange(location: 0, length: text.count), withTemplate: "")
// return modString
// }
// return ""
// })
} else {
readText = ""
}
}
func updateUnreadText() {
// if let textItems = textItems, let item = player?.currentItem as? SpeechPlayerItem {
// let idx = item.speechItem.audioIdx
//
//
// unreadText = Array(textItems[idx...].map { text in
// if let regex = try? NSRegularExpression(pattern: "<[^>]*>", options: .caseInsensitive) {
// let modString = regex.stringByReplacingMatches(in: text, options: [], range: NSRange(location: 0, length: text.count), withTemplate: "")
// return modString
// }
// return ""
// })
// } else {
// unreadText = []
// }
}
public func getPreferredVoice(forLanguage language: String) -> String {
UserDefaults.standard.string(forKey: "\(language)-\(UserDefaultKey.textToSpeechPreferredVoice.rawValue)") ?? currentVoiceLanguage.defaultVoice
}
@ -593,6 +685,13 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
}
player = AVQueuePlayer(items: [])
if let player = player {
observer = player.observe(\.currentItem, options: [.new]) { _, _ in
print("current item did change: ", (player.currentItem as? SpeechPlayerItem)?.speechItem.audioIdx)
self.currentAudioIndex = (player.currentItem as? SpeechPlayerItem)?.speechItem.audioIdx ?? 0
}
}
let synthesizer = SpeechSynthesizer(appEnvironment: appEnvironment, networker: networker, document: document)
durations = synthesizer.estimatedDurations(forSpeed: playbackRate)
self.synthesizer = synthesizer
@ -660,7 +759,7 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
func startTimer() {
if timer == nil {
timer = Timer.scheduledTimer(timeInterval: 0.1, target: self, selector: #selector(fireTimer), userInfo: nil, repeats: true)
timer = Timer.scheduledTimer(timeInterval: 0.2, target: self, selector: #selector(fireTimer), userInfo: nil, repeats: true)
timer?.fire()
}
}
@ -676,6 +775,8 @@ public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate
duration = durations.reduce(0, +)
durationString = formatTimeInterval(duration)
}
updateReadText()
}
if let player = player {

68
apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift
View file

@ -60,8 +60,19 @@ struct SpeechItem {
let audioIdx: Int
let urlRequest: URLRequest
let localAudioURL: URL
let localSpeechURL: URL
let text: String
}
struct SpeechMark: Decodable, Encodable {
let word: String?
let time: Double?
let start: Int?
let length: Int?
}
struct SpeechData {}
struct SpeechSynthesizer {
typealias Element = SpeechItem
let document: SpeechDocument
@ -91,9 +102,15 @@ struct SpeechSynthesizer {
let voiceStr = utterance.voice ?? document.defaultVoice
let segmentStr = String(format: "%04d", arguments: [idx])
let localAudioURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3")
let localSpeechURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).speechMarks")
if let request = urlRequestFor(utterance: utterance) {
let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: idx, urlRequest: request, localAudioURL: localAudioURL)
let item = SpeechItem(htmlIdx: utterance.idx,
audioIdx: idx,
urlRequest: request,
localAudioURL: localAudioURL,
localSpeechURL: localSpeechURL,
text: utterance.text)
return item
}
@ -108,9 +125,15 @@ struct SpeechSynthesizer {
let voiceStr = utterance.voice ?? document.defaultVoice
let segmentStr = String(format: "%04d", arguments: [idx])
let localAudioURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3")
let localSpeechURL = document.audioDirectory.appendingPathComponent("\(segmentStr)-\(voiceStr).mp3")
if let request = urlRequestFor(utterance: utterance) {
let item = SpeechItem(htmlIdx: utterance.idx, audioIdx: idx, urlRequest: request, localAudioURL: localAudioURL)
let item = SpeechItem(htmlIdx: utterance.idx,
audioIdx: idx,
urlRequest: request,
localAudioURL: localAudioURL,
localSpeechURL: localSpeechURL,
text: utterance.text)
result.append(item)
} else {
// TODO: How do we want to handle completely skipped paragraphs?
@ -138,11 +161,17 @@ struct SpeechSynthesizer {
static func download(speechItem: SpeechItem,
redownloadCached: Bool = false,
session: URLSession? = URLSession.shared) async throws -> Data?
session: URLSession? = URLSession.shared) async throws -> SynthesizeData?
{
let decoder = JSONDecoder()
if !redownloadCached, FileManager.default.fileExists(atPath: speechItem.localAudioURL.path) {
if let localData = try? Data(contentsOf: speechItem.localAudioURL) {
return localData
if let speechMarksData = try? Data(contentsOf: speechItem.localSpeechURL),
let speechMarks = try? decoder.decode([SpeechMark].self, from: speechMarksData),
let localData = try? Data(contentsOf: speechItem.localAudioURL)
{
print("CACHED DATA LENGTH: ", localData.count)
// return SynthesizeData(audioData: localData, speechMarks: speechMarks)
}
}
@ -161,20 +190,36 @@ struct SpeechSynthesizer {
.urls(for: .cachesDirectory, in: .userDomainMask)[0]
.appendingPathComponent(UUID().uuidString + ".mp3")
let tempSMPath = FileManager.default
.urls(for: .cachesDirectory, in: .userDomainMask)[0]
.appendingPathComponent(UUID().uuidString + ".speechMarks")
do {
let decoder = JSONDecoder()
let jsonData = try decoder.decode(SynthesizeResult.self, from: data)
print("SPEECH DATA: ", String(decoding: data, as: UTF8.self))
let jsonData = try decoder.decode(SynthesizeResult.self, from: data) as SynthesizeResult
let audioData = Data(fromHexEncodedString: jsonData.audioData)!
if audioData.count < 1 {
throw BasicError.message(messageText: "Audio data is empty")
}
print("AUDIO DATA LENGTH: ", audioData.count)
try audioData.write(to: tempPath)
try? FileManager.default.removeItem(at: speechItem.localAudioURL)
try FileManager.default.moveItem(at: tempPath, to: speechItem.localAudioURL)
return audioData
let encoder = JSONEncoder()
let speechMarksData = try encoder.encode(jsonData.speechMarks)
try speechMarksData.write(to: tempSMPath)
try? FileManager.default.removeItem(at: speechItem.localSpeechURL)
try FileManager.default.moveItem(at: tempSMPath, to: speechItem.localSpeechURL)
print("DOWNLOADED SPEECH MARKS: ", jsonData.speechMarks)
return SynthesizeData(audioData: audioData, speechMarks: jsonData.speechMarks)
} catch {
print("ERROR WRITING DATA", error)
let errorMessage = "audioFetch failed. could not write MP3 data to disk"
throw BasicError.message(messageText: errorMessage)
}
@ -183,7 +228,12 @@ struct SpeechSynthesizer {
struct SynthesizeResult: Decodable {
let audioData: String
// let speechMarks: Any?
let speechMarks: [SpeechMark]
}
struct SynthesizeData: Decodable {
let audioData: Data
let speechMarks: [SpeechMark]
}
extension Data {

8
apple/OmnivoreKit/Sources/Views/Fonts.swift
View file

@ -16,6 +16,14 @@ public extension Font {
/// 22pt, Inter-Regular
static var appTitleTwo: Font {
Font.custom(InterFont.regular.rawValue, size: 28, relativeTo: .title2)
}
static var textToSpeechRead: Font {
Font.custom(InterFont.bold.rawValue, size: 28, relativeTo: .title2)
}
static var textToSpeechUnread: Font {
Font.custom(InterFont.regular.rawValue, size: 22, relativeTo: .title2)
}