Merge branch 'main' of github.com:omnivore-app/omnivore into fix/1096

apple/InfoPlists/Omnivore.plist

@@ -62,6 +62,7 @@
 	<string>Images from your photo library can be chosen by you to send as feedback.</string>
 	<key>UIBackgroundModes</key>
 	<array>
+		<string>audio</string>
 		<string>fetch</string>
 	</array>
 	<key>UILaunchStoryboardName</key>

apple/OmnivoreKit/Sources/App/Services.swift

@@ -12,11 +12,13 @@ public final class Services {
 
   public let authenticator: Authenticator
   public let dataService: DataService
+  public let audioSession: AudioSession
 
   public init(appEnvironment: AppEnvironment = PublicValet.storedAppEnvironment ?? .initialAppEnvironment) {
     let networker = Networker(appEnvironment: appEnvironment)
     self.authenticator = Authenticator(networker: networker)
     self.dataService = DataService(appEnvironment: appEnvironment, networker: networker)
+    self.audioSession = AudioSession(appEnvironment: appEnvironment, networker: networker)
   }
 }
 

apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift

@@ -0,0 +1,100 @@
+//
+//  MiniPlayer.swift
+//
+//
+//  Created by Jackson Harper on 8/15/22.
+//
+
+import Foundation
+import Services
+import SwiftUI
+
+public struct MiniPlayer: View {
+  @EnvironmentObject var audioSession: AudioSession
+  @Environment(\.colorScheme) private var colorScheme: ColorScheme
+
+  private let presentingView: AnyView
+
+  init<PresentingView>(
+    presentingView: PresentingView
+  ) where PresentingView: View {
+    self.presentingView = AnyView(presentingView)
+  }
+
+  var playPauseButtonItem: some View {
+    if let item = audioSession.item, audioSession.isLoadingItem(item: item) {
+      return AnyView(ProgressView())
+    } else {
+      return AnyView(Button(
+        action: {
+          switch audioSession.state {
+          case .playing:
+            audioSession.pause()
+          case .paused:
+            audioSession.unpause()
+          default:
+            break
+          }
+        },
+        label: {
+          Image(systemName: audioSession.state == .playing ? "pause.circle" : "play.circle")
+            .font(.appTitleTwo)
+        }
+      ))
+    }
+  }
+
+  var stopButton: some View {
+    Button(
+      action: {
+        audioSession.stop()
+      },
+      label: {
+        Image(systemName: "xmark")
+          .font(.appTitleTwo)
+      }
+    )
+  }
+
+  public var body: some View {
+    GeometryReader { geometry in
+      ZStack(alignment: .center) {
+        presentingView
+        VStack {
+          Spacer()
+          if let item = audioSession.item, self.audioSession.state != .stopped {
+            HStack {
+              Text(item.unwrappedTitle)
+                .font(.appCallout)
+                .lineSpacing(1.25)
+                .foregroundColor(.appGrayTextContrast)
+                .fixedSize(horizontal: false, vertical: true)
+                .frame(maxWidth: .infinity, alignment: .leading)
+
+              playPauseButtonItem
+                .frame(width: 28, height: 28)
+
+              stopButton
+                .frame(width: 28, height: 28)
+            }
+            .padding()
+            .frame(width: geometry.size.width, height: 88) // this should be 108 once we add GrabberVisible at the bottom
+            .animation(.spring(), value: true)
+            .tint(.appGrayTextContrast)
+            .background(
+              Color.systemBackground
+                .shadow(color: .gray.opacity(0.33), radius: 8, x: 0, y: 4)
+                .mask(Rectangle().padding(.top, -20))
+            )
+          }
+        }
+      }
+    }
+  }
+}
+
+public extension View {
+  func miniPlayer() -> some View {
+    MiniPlayer(presentingView: self)
+  }
+}

apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift

@@ -27,6 +27,7 @@ public struct RootView: View {
     InnerRootView(viewModel: viewModel)
       .environmentObject(viewModel.services.authenticator)
       .environmentObject(viewModel.services.dataService)
+      .environmentObject(viewModel.services.audioSession)
       .environment(\.managedObjectContext, viewModel.services.dataService.viewContext)
       .onChange(of: scenePhase) { phase in
         if phase == .background {
@@ -50,6 +51,7 @@ struct InnerRootView: View {
         .onAppear {
           viewModel.triggerPushNotificationRequestIfNeeded()
         }
+        .miniPlayer()
         .snackBar(isShowing: $viewModel.showSnackbar, message: viewModel.snackbarMessage)
         // Schedule the dismissal every time we present the snackbar.
         .onChange(of: viewModel.showSnackbar) { newValue in

apple/OmnivoreKit/Sources/App/Views/RootView/RootViewModel.swift

@@ -17,6 +17,7 @@ public final class RootViewModel: ObservableObject {
   @Published public var showPushNotificationPrimer = false
   @Published var snackbarMessage: String?
   @Published var showSnackbar = false
+  @Published var showMiniPlayer = true
 
   public init() {
     registerFonts()

apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift

@@ -1,6 +1,8 @@
+import AVFoundation
 import Models
 import Services
 import SwiftUI
+import Utils
 import Views
 import WebKit
 
@@ -22,6 +24,7 @@ struct WebReaderContainerView: View {
   @State var annotation = String()
 
   @EnvironmentObject var dataService: DataService
+  @EnvironmentObject var audioSession: AudioSession
   @Environment(\.presentationMode) var presentationMode: Binding<PresentationMode>
   @StateObject var viewModel = WebReaderViewModel()
 
@@ -53,6 +56,41 @@ struct WebReaderContainerView: View {
     }
   }
 
+  var audioNavbarItem: some View {
+    if audioSession.isLoadingItem(item: item) {
+      return AnyView(ProgressView()
+        .padding(.horizontal)
+        .scaleEffect(navBarVisibilityRatio))
+    } else {
+      return AnyView(Button(
+        action: {
+          switch audioSession.state {
+          case .playing:
+            if audioSession.item == self.item {
+              audioSession.pause()
+              return
+            }
+            fallthrough
+          case .paused:
+            if audioSession.item == self.item {
+              audioSession.unpause()
+              return
+            }
+            fallthrough
+          default:
+            audioSession.play(item: self.item)
+          }
+        },
+        label: {
+          Image(systemName: audioSession.isPlayingItem(item: item) ? "pause.circle" : "play.circle")
+            .font(.appTitleTwo)
+        }
+      )
+      .padding(.horizontal)
+      .scaleEffect(navBarVisibilityRatio))
+    }
+  }
+
   var navBar: some View {
     HStack(alignment: .center) {
       #if os(iOS)
@@ -68,6 +106,9 @@ struct WebReaderContainerView: View {
         .scaleEffect(navBarVisibilityRatio)
         Spacer()
       #endif
+      if FeatureFlag.enableTextToSpeechButton {
+        audioNavbarItem
+      }
       Button(
         action: { showPreferencesPopover.toggle() },
         label: {

apple/OmnivoreKit/Sources/Services/AudioSession/AudioSession.swift

@@ -0,0 +1,255 @@
+//
+//  AudioSession.swift
+//
+//
+//  Created by Jackson Harper on 8/15/22.
+//
+
+import AVFoundation
+import CryptoKit
+import Foundation
+import MediaPlayer
+import Models
+import Utils
+
+public enum AudioSessionState {
+  case stopped
+  case paused
+  case loading
+  case playing
+}
+
+// Our observable object class
+public class AudioSession: ObservableObject {
+  @Published public var state: AudioSessionState = .stopped
+  @Published public var item: LinkedItem?
+
+  let appEnvironment: AppEnvironment
+  let networker: Networker
+
+  var timer: Timer?
+  var player: AVAudioPlayer?
+
+  public init(appEnvironment: AppEnvironment, networker: Networker) {
+    self.appEnvironment = appEnvironment
+    self.networker = networker
+  }
+
+  public func play(item: LinkedItem) {
+    // Stop any existing session
+    stop()
+
+    self.item = item
+    startAudio()
+  }
+
+  public func stop() {
+    player?.stop()
+    clearNowPlayingInfo()
+    timer = nil
+    player = nil
+    item = nil
+    state = .stopped
+  }
+
+  public func isLoadingItem(item: LinkedItem) -> Bool {
+    state == .loading && self.item == item
+  }
+
+  public func isPlayingItem(item: LinkedItem) -> Bool {
+    state == .playing && self.item == item
+  }
+
+  public func startAudio() {
+    state = .loading
+
+    let pageId = item!.unwrappedID
+
+    Task {
+      do {
+        _ = try await downloadAudioFile(pageId: pageId)
+        DispatchQueue.main.async {
+          self.startDownloadedAudioFile(pageId: pageId)
+        }
+      } catch {
+        // TODO: maybe we need a failed state?
+        DispatchQueue.main.async {
+          self.state = .stopped
+        }
+        print("FAILED TO DOWNLOAD AUDIO URL")
+        print(error.localizedDescription)
+      }
+    }
+  }
+
+  private func startDownloadedAudioFile(pageId: String) {
+    // Make sure audio file is still correct for the current page
+    guard item?.unwrappedID == pageId else {
+      state = .stopped
+      return
+    }
+
+    // TODO: Maybe check if app is active so it doesn't end up playing later?
+
+    let audioUrl = FileManager.default
+      .urls(for: .documentDirectory, in: .userDomainMask)[0]
+      .appendingPathComponent(pageId + ".mp3")
+
+    if !FileManager.default.fileExists(atPath: audioUrl.path) {
+      stop()
+      return
+    }
+
+    do {
+      try AVAudioSession.sharedInstance().setCategory(.playback)
+
+      player = try AVAudioPlayer(contentsOf: audioUrl)
+      if player?.play() ?? false {
+        state = .playing
+        startTimer()
+        setupRemoteControl()
+      }
+    } catch {
+      print("error playing MP3 file", error)
+      print(error.localizedDescription)
+      state = .stopped
+    }
+  }
+
+  public func pause() -> Bool {
+    if let player = player {
+      player.pause()
+      state = .paused
+      return true
+    }
+    return false
+  }
+
+  public func unpause() -> Bool {
+    playAudio()
+  }
+
+  public func playAudio() -> Bool {
+    if let player = player {
+      player.play()
+      state = .playing
+      return true
+    }
+    return false
+  }
+
+  func startTimer() {
+    if timer == nil {
+      // Update every 100ms
+      timer = Timer.scheduledTimer(timeInterval: 0.1, target: self, selector: #selector(update(_:)), userInfo: nil, repeats: true)
+      timer?.fire()
+    }
+  }
+
+  func stopTimer() {
+    timer = nil
+  }
+
+  // Every second, get the current playing time of the player and refresh the status of the player progressslider
+  @objc func update(_: Timer) {
+    if let player = player, player.isPlaying {
+      print("play time in ms: ", Int(player.currentTime * 1000))
+    }
+  }
+
+  func clearNowPlayingInfo() {
+    MPNowPlayingInfoCenter.default().nowPlayingInfo = [:]
+  }
+
+  func setupRemoteControl() {
+    UIApplication.shared.beginReceivingRemoteControlEvents()
+
+    MPNowPlayingInfoCenter.default().nowPlayingInfo = [
+      //  MPMediaItemArtwork: ""m
+      MPMediaItemPropertyArtist: item?.author ?? "Omnivore",
+      MPMediaItemPropertyTitle: item?.title ?? "Your Omnivore Article"
+    ]
+
+    if let imageURL = item?.imageURL, let cachedImage = ImageCache.shared[imageURL] {
+//      #if os(iOS)
+//        status = .loaded(image: Image(uiImage: cachedImage))
+//      #else
+//        status = .loaded(image: Image(nsImage: cachedImage))
+//      #endif
+      MPNowPlayingInfoCenter.default().nowPlayingInfo = [
+        //  MPMediaItemPropertyArtwork: cachedImage,
+        MPMediaItemPropertyArtist: item?.author ?? "Omnivore",
+        MPMediaItemPropertyTitle: item?.title ?? "Your Omnivore Article"
+      ]
+    }
+
+    let commandCenter = MPRemoteCommandCenter.shared()
+
+    commandCenter.playCommand.isEnabled = true
+    commandCenter.playCommand.addTarget { _ -> MPRemoteCommandHandlerStatus in
+      self.unpause()
+      return .success
+    }
+
+    commandCenter.pauseCommand.isEnabled = true
+    commandCenter.pauseCommand.addTarget { _ -> MPRemoteCommandHandlerStatus in
+      self.pause()
+      return .success
+    }
+  }
+
+  func downloadAudioFile(pageId: String) async throws -> URL? {
+    let audioUrl = FileManager.default
+      .urls(for: .documentDirectory, in: .userDomainMask)[0]
+      .appendingPathComponent(pageId + ".mp3")
+
+//    if FileManager.default.fileExists(atPath: audioUrl.path) {
+//      // Prevent re-download
+//      // TODO: We aren't doing this very safely, we should be verifying a checksum
+//      return audioUrl
+//    }
+
+    guard let url = URL(string: "/api/article/\(pageId)/mp3", relativeTo: appEnvironment.serverBaseURL) else {
+      throw BasicError.message(messageText: "Invalid audio URL")
+    }
+
+    var request = URLRequest(url: url)
+    request.httpMethod = "GET"
+    request.timeoutInterval = 600
+    for (header, value) in networker.defaultHeaders {
+      request.setValue(value, forHTTPHeaderField: header)
+    }
+
+    let result: (Data, URLResponse)? = try? await URLSession.shared.data(for: request)
+    guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else {
+      throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.")
+    }
+    print("httpResponse: ", httpResponse)
+
+    guard let data = result?.0 else {
+      throw BasicError.message(messageText: "audioFetch failed. no data received.")
+    }
+
+    let tempPath = FileManager.default
+      .urls(for: .cachesDirectory, in: .userDomainMask)[0]
+      .appendingPathComponent(UUID().uuidString + ".mp3")
+
+    do {
+      if let googleHash = httpResponse.value(forHTTPHeaderField: "x-goog-hash") {
+        let hash = Data(Insecure.MD5.hash(data: data)).base64EncodedString()
+        if !googleHash.contains("md5=\(hash)") {
+          print("Downloaded mp3 file hashes do not match: returned: \(googleHash) v computed: \(hash)")
+          throw BasicError.message(messageText: "Downloaded mp3 file hashes do not match: returned: \(googleHash) v computed: \(hash)")
+        }
+      }
+
+      try data.write(to: tempPath)
+      try FileManager.default.moveItem(at: tempPath, to: audioUrl)
+    } catch {
+      let errorMessage = "audioFetch failed. could not write MP3 data to disk"
+      throw BasicError.message(messageText: errorMessage)
+    }
+
+    return audioUrl
+  }
+}

apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift

@@ -14,4 +14,5 @@ public enum FeatureFlag {
   public static let enableShareButton = false
   public static let enableSnooze = false
   public static let enableGridCardsOnPhone = false
+  public static let enableTextToSpeechButton = true
 }

apple/OmnivoreKit/Sources/Utils/ImageCache.swift

@@ -27,16 +27,16 @@ public final class ImageCache {
   }
 
   private let queue = DispatchQueue(label: "app.omnivore.image.cache.queue", attributes: .concurrent)
-  private let cache = NSCache<AnyObject, PlatformImage>()
+  private let cache = NSCache<NSString, PlatformImage>()
 
   private init() {
-    cache.totalCostLimit = 1024 * 1024 * 50 // 50 MB
+    cache.totalCostLimit = 1024 * 1024 * 1024 * 50 // 50 MB
   }
 
   private func image(_ url: URL) -> PlatformImage? {
     var cachedImage: PlatformImage?
     queue.sync {
-      cachedImage = cache.object(forKey: url as AnyObject)
+      cachedImage = cache.object(forKey: NSString(string: url.absoluteString))
     }
     return cachedImage
   }
@@ -44,7 +44,7 @@ public final class ImageCache {
   private func insertImage(_ image: PlatformImage?, url: URL) {
     guard let image = image else { return }
     queue.async(flags: .barrier) {
-      self.cache.setObject(image, forKey: url as AnyObject, cost: image.diskSize)
+      self.cache.setObject(image, forKey: NSString(string: url.absoluteString), cost: 1)
     }
   }
 }

packages/api/src/elastic/labels.ts

@@ -186,7 +186,7 @@ export const updateLabel = async (
                      ctx._source.labels.add(params.label);
                      ctx._source.updatedAt = params.updatedAt
                    }
-                   if (ctx._source.highlights != null) {
+                   if (ctx._source.highlights != null && ctx._source.highlights[0].labels != null) {
                      ctx._source.highlights[0].labels.removeIf(l -> l.id == params.label.id);
                      ctx._source.highlights[0].labels.add(params.label);
                      ctx._source.updatedAt = params.updatedAt

packages/api/src/routers/article_router.ts

@@ -155,8 +155,8 @@ export function articleRouter() {
         // update state
         await getRepository(Speech).update(speech.id, {
           state: SpeechState.COMPLETED,
-          audioFileName: speech.audioFileName,
-          speechMarksFileName: speech.speechMarksFileName,
+          audioFileName: speechOutput.audioFileName,
+          speechMarksFileName: speechOutput.speechMarksFileName,
         })
         speech.audioFileName = speechOutput.audioFileName
         speech.speechMarksFileName = speechOutput.speechMarksFileName

packages/web/components/templates/library/LibraryMenu.tsx

@@ -12,8 +12,10 @@ export function LibraryMenu(): JSX.Element {
       css={{
         width: '286px',
         minWidth: '286px',
+        pl:'15px',
         height: 'calc(100% - 100px)',
         overflowY: 'auto',
+        fontWeight: '600',
       }}
     >
       <Menubar />