diff --git a/apple/InfoPlists/Omnivore.plist b/apple/InfoPlists/Omnivore.plist
index c9a47cec4..2495872c4 100644
--- a/apple/InfoPlists/Omnivore.plist
+++ b/apple/InfoPlists/Omnivore.plist
@@ -62,6 +62,7 @@
Images from your photo library can be chosen by you to send as feedback.
UIBackgroundModes
+ audio
fetch
UILaunchStoryboardName
diff --git a/apple/OmnivoreKit/Sources/App/Services.swift b/apple/OmnivoreKit/Sources/App/Services.swift
index b7b602c17..e7d9fe31b 100644
--- a/apple/OmnivoreKit/Sources/App/Services.swift
+++ b/apple/OmnivoreKit/Sources/App/Services.swift
@@ -12,11 +12,13 @@ public final class Services {
public let authenticator: Authenticator
public let dataService: DataService
+ public let audioSession: AudioSession
public init(appEnvironment: AppEnvironment = PublicValet.storedAppEnvironment ?? .initialAppEnvironment) {
let networker = Networker(appEnvironment: appEnvironment)
self.authenticator = Authenticator(networker: networker)
self.dataService = DataService(appEnvironment: appEnvironment, networker: networker)
+ self.audioSession = AudioSession(appEnvironment: appEnvironment, networker: networker)
}
}
diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift
new file mode 100644
index 000000000..bc2e8498f
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift
@@ -0,0 +1,100 @@
+//
+// MiniPlayer.swift
+//
+//
+// Created by Jackson Harper on 8/15/22.
+//
+
+import Foundation
+import Services
+import SwiftUI
+
+public struct MiniPlayer: View {
+ @EnvironmentObject var audioSession: AudioSession
+ @Environment(\.colorScheme) private var colorScheme: ColorScheme
+
+ private let presentingView: AnyView
+
+ init(
+ presentingView: PresentingView
+ ) where PresentingView: View {
+ self.presentingView = AnyView(presentingView)
+ }
+
+ var playPauseButtonItem: some View {
+ if let item = audioSession.item, audioSession.isLoadingItem(item: item) {
+ return AnyView(ProgressView())
+ } else {
+ return AnyView(Button(
+ action: {
+ switch audioSession.state {
+ case .playing:
+ audioSession.pause()
+ case .paused:
+ audioSession.unpause()
+ default:
+ break
+ }
+ },
+ label: {
+ Image(systemName: audioSession.state == .playing ? "pause.circle" : "play.circle")
+ .font(.appTitleTwo)
+ }
+ ))
+ }
+ }
+
+ var stopButton: some View {
+ Button(
+ action: {
+ audioSession.stop()
+ },
+ label: {
+ Image(systemName: "xmark")
+ .font(.appTitleTwo)
+ }
+ )
+ }
+
+ public var body: some View {
+ GeometryReader { geometry in
+ ZStack(alignment: .center) {
+ presentingView
+ VStack {
+ Spacer()
+ if let item = audioSession.item, self.audioSession.state != .stopped {
+ HStack {
+ Text(item.unwrappedTitle)
+ .font(.appCallout)
+ .lineSpacing(1.25)
+ .foregroundColor(.appGrayTextContrast)
+ .fixedSize(horizontal: false, vertical: true)
+ .frame(maxWidth: .infinity, alignment: .leading)
+
+ playPauseButtonItem
+ .frame(width: 28, height: 28)
+
+ stopButton
+ .frame(width: 28, height: 28)
+ }
+ .padding()
+ .frame(width: geometry.size.width, height: 88) // this should be 108 once we add GrabberVisible at the bottom
+ .animation(.spring(), value: true)
+ .tint(.appGrayTextContrast)
+ .background(
+ Color.systemBackground
+ .shadow(color: .gray.opacity(0.33), radius: 8, x: 0, y: 4)
+ .mask(Rectangle().padding(.top, -20))
+ )
+ }
+ }
+ }
+ }
+ }
+}
+
+public extension View {
+ func miniPlayer() -> some View {
+ MiniPlayer(presentingView: self)
+ }
+}
diff --git a/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift b/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift
index 77f6c2982..c603f4ff2 100644
--- a/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift
@@ -27,6 +27,7 @@ public struct RootView: View {
InnerRootView(viewModel: viewModel)
.environmentObject(viewModel.services.authenticator)
.environmentObject(viewModel.services.dataService)
+ .environmentObject(viewModel.services.audioSession)
.environment(\.managedObjectContext, viewModel.services.dataService.viewContext)
.onChange(of: scenePhase) { phase in
if phase == .background {
@@ -50,6 +51,7 @@ struct InnerRootView: View {
.onAppear {
viewModel.triggerPushNotificationRequestIfNeeded()
}
+ .miniPlayer()
.snackBar(isShowing: $viewModel.showSnackbar, message: viewModel.snackbarMessage)
// Schedule the dismissal every time we present the snackbar.
.onChange(of: viewModel.showSnackbar) { newValue in
diff --git a/apple/OmnivoreKit/Sources/App/Views/RootView/RootViewModel.swift b/apple/OmnivoreKit/Sources/App/Views/RootView/RootViewModel.swift
index 033f6f5e4..92eec8ce0 100644
--- a/apple/OmnivoreKit/Sources/App/Views/RootView/RootViewModel.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/RootView/RootViewModel.swift
@@ -17,6 +17,7 @@ public final class RootViewModel: ObservableObject {
@Published public var showPushNotificationPrimer = false
@Published var snackbarMessage: String?
@Published var showSnackbar = false
+ @Published var showMiniPlayer = true
public init() {
registerFonts()
diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
index e12ba8b07..8856ac072 100644
--- a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
+++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
@@ -1,6 +1,8 @@
+import AVFoundation
import Models
import Services
import SwiftUI
+import Utils
import Views
import WebKit
@@ -22,6 +24,7 @@ struct WebReaderContainerView: View {
@State var annotation = String()
@EnvironmentObject var dataService: DataService
+ @EnvironmentObject var audioSession: AudioSession
@Environment(\.presentationMode) var presentationMode: Binding
@StateObject var viewModel = WebReaderViewModel()
@@ -53,6 +56,41 @@ struct WebReaderContainerView: View {
}
}
+ var audioNavbarItem: some View {
+ if audioSession.isLoadingItem(item: item) {
+ return AnyView(ProgressView()
+ .padding(.horizontal)
+ .scaleEffect(navBarVisibilityRatio))
+ } else {
+ return AnyView(Button(
+ action: {
+ switch audioSession.state {
+ case .playing:
+ if audioSession.item == self.item {
+ audioSession.pause()
+ return
+ }
+ fallthrough
+ case .paused:
+ if audioSession.item == self.item {
+ audioSession.unpause()
+ return
+ }
+ fallthrough
+ default:
+ audioSession.play(item: self.item)
+ }
+ },
+ label: {
+ Image(systemName: audioSession.isPlayingItem(item: item) ? "pause.circle" : "play.circle")
+ .font(.appTitleTwo)
+ }
+ )
+ .padding(.horizontal)
+ .scaleEffect(navBarVisibilityRatio))
+ }
+ }
+
var navBar: some View {
HStack(alignment: .center) {
#if os(iOS)
@@ -68,6 +106,9 @@ struct WebReaderContainerView: View {
.scaleEffect(navBarVisibilityRatio)
Spacer()
#endif
+ if FeatureFlag.enableTextToSpeechButton {
+ audioNavbarItem
+ }
Button(
action: { showPreferencesPopover.toggle() },
label: {
diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift
new file mode 100644
index 000000000..9185fef59
--- /dev/null
+++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift
@@ -0,0 +1,255 @@
+//
+// AudioSession.swift
+//
+//
+// Created by Jackson Harper on 8/15/22.
+//
+
+import AVFoundation
+import CryptoKit
+import Foundation
+import MediaPlayer
+import Models
+import Utils
+
+public enum AudioSessionState {
+ case stopped
+ case paused
+ case loading
+ case playing
+}
+
+// Our observable object class
+public class AudioSession: ObservableObject {
+ @Published public var state: AudioSessionState = .stopped
+ @Published public var item: LinkedItem?
+
+ let appEnvironment: AppEnvironment
+ let networker: Networker
+
+ var timer: Timer?
+ var player: AVAudioPlayer?
+
+ public init(appEnvironment: AppEnvironment, networker: Networker) {
+ self.appEnvironment = appEnvironment
+ self.networker = networker
+ }
+
+ public func play(item: LinkedItem) {
+ // Stop any existing session
+ stop()
+
+ self.item = item
+ startAudio()
+ }
+
+ public func stop() {
+ player?.stop()
+ clearNowPlayingInfo()
+ timer = nil
+ player = nil
+ item = nil
+ state = .stopped
+ }
+
+ public func isLoadingItem(item: LinkedItem) -> Bool {
+ state == .loading && self.item == item
+ }
+
+ public func isPlayingItem(item: LinkedItem) -> Bool {
+ state == .playing && self.item == item
+ }
+
+ public func startAudio() {
+ state = .loading
+
+ let pageId = item!.unwrappedID
+
+ Task {
+ do {
+ _ = try await downloadAudioFile(pageId: pageId)
+ DispatchQueue.main.async {
+ self.startDownloadedAudioFile(pageId: pageId)
+ }
+ } catch {
+ // TODO: maybe we need a failed state?
+ DispatchQueue.main.async {
+ self.state = .stopped
+ }
+ print("FAILED TO DOWNLOAD AUDIO URL")
+ print(error.localizedDescription)
+ }
+ }
+ }
+
+ private func startDownloadedAudioFile(pageId: String) {
+ // Make sure audio file is still correct for the current page
+ guard item?.unwrappedID == pageId else {
+ state = .stopped
+ return
+ }
+
+ // TODO: Maybe check if app is active so it doesn't end up playing later?
+
+ let audioUrl = FileManager.default
+ .urls(for: .documentDirectory, in: .userDomainMask)[0]
+ .appendingPathComponent(pageId + ".mp3")
+
+ if !FileManager.default.fileExists(atPath: audioUrl.path) {
+ stop()
+ return
+ }
+
+ do {
+ try AVAudioSession.sharedInstance().setCategory(.playback)
+
+ player = try AVAudioPlayer(contentsOf: audioUrl)
+ if player?.play() ?? false {
+ state = .playing
+ startTimer()
+ setupRemoteControl()
+ }
+ } catch {
+ print("error playing MP3 file", error)
+ print(error.localizedDescription)
+ state = .stopped
+ }
+ }
+
+ public func pause() -> Bool {
+ if let player = player {
+ player.pause()
+ state = .paused
+ return true
+ }
+ return false
+ }
+
+ public func unpause() -> Bool {
+ playAudio()
+ }
+
+ public func playAudio() -> Bool {
+ if let player = player {
+ player.play()
+ state = .playing
+ return true
+ }
+ return false
+ }
+
+ func startTimer() {
+ if timer == nil {
+ // Update every 100ms
+ timer = Timer.scheduledTimer(timeInterval: 0.1, target: self, selector: #selector(update(_:)), userInfo: nil, repeats: true)
+ timer?.fire()
+ }
+ }
+
+ func stopTimer() {
+ timer = nil
+ }
+
+ // Every second, get the current playing time of the player and refresh the status of the player progressslider
+ @objc func update(_: Timer) {
+ if let player = player, player.isPlaying {
+ print("play time in ms: ", Int(player.currentTime * 1000))
+ }
+ }
+
+ func clearNowPlayingInfo() {
+ MPNowPlayingInfoCenter.default().nowPlayingInfo = [:]
+ }
+
+ func setupRemoteControl() {
+ UIApplication.shared.beginReceivingRemoteControlEvents()
+
+ MPNowPlayingInfoCenter.default().nowPlayingInfo = [
+ // MPMediaItemArtwork: ""m
+ MPMediaItemPropertyArtist: item?.author ?? "Omnivore",
+ MPMediaItemPropertyTitle: item?.title ?? "Your Omnivore Article"
+ ]
+
+ if let imageURL = item?.imageURL, let cachedImage = ImageCache.shared[imageURL] {
+// #if os(iOS)
+// status = .loaded(image: Image(uiImage: cachedImage))
+// #else
+// status = .loaded(image: Image(nsImage: cachedImage))
+// #endif
+ MPNowPlayingInfoCenter.default().nowPlayingInfo = [
+ // MPMediaItemPropertyArtwork: cachedImage,
+ MPMediaItemPropertyArtist: item?.author ?? "Omnivore",
+ MPMediaItemPropertyTitle: item?.title ?? "Your Omnivore Article"
+ ]
+ }
+
+ let commandCenter = MPRemoteCommandCenter.shared()
+
+ commandCenter.playCommand.isEnabled = true
+ commandCenter.playCommand.addTarget { _ -> MPRemoteCommandHandlerStatus in
+ self.unpause()
+ return .success
+ }
+
+ commandCenter.pauseCommand.isEnabled = true
+ commandCenter.pauseCommand.addTarget { _ -> MPRemoteCommandHandlerStatus in
+ self.pause()
+ return .success
+ }
+ }
+
+ func downloadAudioFile(pageId: String) async throws -> URL? {
+ let audioUrl = FileManager.default
+ .urls(for: .documentDirectory, in: .userDomainMask)[0]
+ .appendingPathComponent(pageId + ".mp3")
+
+// if FileManager.default.fileExists(atPath: audioUrl.path) {
+// // Prevent re-download
+// // TODO: We aren't doing this very safely, we should be verifying a checksum
+// return audioUrl
+// }
+
+ guard let url = URL(string: "/api/article/\(pageId)/mp3", relativeTo: appEnvironment.serverBaseURL) else {
+ throw BasicError.message(messageText: "Invalid audio URL")
+ }
+
+ var request = URLRequest(url: url)
+ request.httpMethod = "GET"
+ request.timeoutInterval = 600
+ for (header, value) in networker.defaultHeaders {
+ request.setValue(value, forHTTPHeaderField: header)
+ }
+
+ let result: (Data, URLResponse)? = try? await URLSession.shared.data(for: request)
+ guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else {
+ throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.")
+ }
+ print("httpResponse: ", httpResponse)
+
+ guard let data = result?.0 else {
+ throw BasicError.message(messageText: "audioFetch failed. no data received.")
+ }
+
+ let tempPath = FileManager.default
+ .urls(for: .cachesDirectory, in: .userDomainMask)[0]
+ .appendingPathComponent(UUID().uuidString + ".mp3")
+
+ do {
+ if let googleHash = httpResponse.value(forHTTPHeaderField: "x-goog-hash") {
+ let hash = Data(Insecure.MD5.hash(data: data)).base64EncodedString()
+ if !googleHash.contains("md5=\(hash)") {
+ print("Downloaded mp3 file hashes do not match: returned: \(googleHash) v computed: \(hash)")
+ throw BasicError.message(messageText: "Downloaded mp3 file hashes do not match: returned: \(googleHash) v computed: \(hash)")
+ }
+ }
+
+ try data.write(to: tempPath)
+ try FileManager.default.moveItem(at: tempPath, to: audioUrl)
+ } catch {
+ let errorMessage = "audioFetch failed. could not write MP3 data to disk"
+ throw BasicError.message(messageText: errorMessage)
+ }
+
+ return audioUrl
+ }
+}
diff --git a/apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift b/apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift
index 5c0aab9af..5ace227a6 100644
--- a/apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift
+++ b/apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift
@@ -14,4 +14,5 @@ public enum FeatureFlag {
public static let enableShareButton = false
public static let enableSnooze = false
public static let enableGridCardsOnPhone = false
+ public static let enableTextToSpeechButton = true
}
diff --git a/apple/OmnivoreKit/Sources/Utils/ImageCache.swift b/apple/OmnivoreKit/Sources/Utils/ImageCache.swift
index 1e8e4416b..f05003969 100644
--- a/apple/OmnivoreKit/Sources/Utils/ImageCache.swift
+++ b/apple/OmnivoreKit/Sources/Utils/ImageCache.swift
@@ -27,16 +27,16 @@ public final class ImageCache {
}
private let queue = DispatchQueue(label: "app.omnivore.image.cache.queue", attributes: .concurrent)
- private let cache = NSCache()
+ private let cache = NSCache()
private init() {
- cache.totalCostLimit = 1024 * 1024 * 50 // 50 MB
+ cache.totalCostLimit = 1024 * 1024 * 1024 * 50 // 50 MB
}
private func image(_ url: URL) -> PlatformImage? {
var cachedImage: PlatformImage?
queue.sync {
- cachedImage = cache.object(forKey: url as AnyObject)
+ cachedImage = cache.object(forKey: NSString(string: url.absoluteString))
}
return cachedImage
}
@@ -44,7 +44,7 @@ public final class ImageCache {
private func insertImage(_ image: PlatformImage?, url: URL) {
guard let image = image else { return }
queue.async(flags: .barrier) {
- self.cache.setObject(image, forKey: url as AnyObject, cost: image.diskSize)
+ self.cache.setObject(image, forKey: NSString(string: url.absoluteString), cost: 1)
}
}
}
diff --git a/packages/api/src/elastic/labels.ts b/packages/api/src/elastic/labels.ts
index 3c1423f7c..d5f931553 100644
--- a/packages/api/src/elastic/labels.ts
+++ b/packages/api/src/elastic/labels.ts
@@ -186,7 +186,7 @@ export const updateLabel = async (
ctx._source.labels.add(params.label);
ctx._source.updatedAt = params.updatedAt
}
- if (ctx._source.highlights != null) {
+ if (ctx._source.highlights != null && ctx._source.highlights[0].labels != null) {
ctx._source.highlights[0].labels.removeIf(l -> l.id == params.label.id);
ctx._source.highlights[0].labels.add(params.label);
ctx._source.updatedAt = params.updatedAt
diff --git a/packages/api/src/routers/article_router.ts b/packages/api/src/routers/article_router.ts
index 25731076a..31fbc2eac 100644
--- a/packages/api/src/routers/article_router.ts
+++ b/packages/api/src/routers/article_router.ts
@@ -155,8 +155,8 @@ export function articleRouter() {
// update state
await getRepository(Speech).update(speech.id, {
state: SpeechState.COMPLETED,
- audioFileName: speech.audioFileName,
- speechMarksFileName: speech.speechMarksFileName,
+ audioFileName: speechOutput.audioFileName,
+ speechMarksFileName: speechOutput.speechMarksFileName,
})
speech.audioFileName = speechOutput.audioFileName
speech.speechMarksFileName = speechOutput.speechMarksFileName
diff --git a/packages/web/components/templates/library/LibraryMenu.tsx b/packages/web/components/templates/library/LibraryMenu.tsx
index 1d9808e52..b1ffaa9b6 100644
--- a/packages/web/components/templates/library/LibraryMenu.tsx
+++ b/packages/web/components/templates/library/LibraryMenu.tsx
@@ -12,8 +12,10 @@ export function LibraryMenu(): JSX.Element {
css={{
width: '286px',
minWidth: '286px',
+ pl:'15px',
height: 'calc(100% - 100px)',
overflowY: 'auto',
+ fontWeight: '600',
}}
>