Merge pull request #3093 from omnivore-app/feat/ios-update-voices

Update voices

apple/Omnivore.xcodeproj/project.pbxproj

@@ -1400,7 +1400,7 @@
 					"@executable_path/../Frameworks",
 				);
 				MACOSX_DEPLOYMENT_TARGET = 12.0;
-				MARKETING_VERSION = 1.36.0;
+				MARKETING_VERSION = 1.37.0;
 				MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
@@ -1435,7 +1435,7 @@
 					"@executable_path/../Frameworks",
 				);
 				MACOSX_DEPLOYMENT_TARGET = 12.0;
-				MARKETING_VERSION = 1.36.0;
+				MARKETING_VERSION = 1.37.0;
 				MTL_FAST_MATH = YES;
 				PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
 				PRODUCT_NAME = "$(TARGET_NAME)";
@@ -1490,7 +1490,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 1.36.0;
+				MARKETING_VERSION = 1.37.0;
 				PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
 				PRODUCT_NAME = Omnivore;
 				PROVISIONING_PROFILE_SPECIFIER = "";
@@ -1831,7 +1831,7 @@
 					"$(inherited)",
 					"@executable_path/Frameworks",
 				);
-				MARKETING_VERSION = 1.36.0;
+				MARKETING_VERSION = 1.37.0;
 				PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
 				PRODUCT_NAME = Omnivore;
 				PROVISIONING_PROFILE_SPECIFIER = "";

apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ExpandedPlayer.swift

@@ -28,6 +28,9 @@
     }
 
     var playPauseButtonItem: some View {
+      if audioController.playbackError {
+        return AnyView(Color.clear)
+      }
       if let itemID = audioController.itemAudioProperties?.itemID, audioController.isLoadingItem(itemID: itemID) {
         return AnyView(ProgressView())
       } else {
@@ -349,6 +352,9 @@
 
     func playerContent(_: LinkedItemAudioProperties) -> some View {
       ZStack {
+        if audioController.playbackError {
+          Text("There was an error playing back your audio.").foregroundColor(Color.red).font(.footnote)
+        }
         audioCards
           .frame(maxHeight: .infinity)
 
@@ -422,12 +428,10 @@
 
     public var innerBody: some View {
       if let itemAudioProperties = self.audioController.itemAudioProperties {
-        return AnyView(playerContent(itemAudioProperties)
-          .tint(.appGrayTextContrast)
-          .alert("There was an error playing back your audio.",
-                 isPresented: $audioController.playbackError) {
-            Button(LocalText.dismissButton, role: .none) {}
-          })
+        return AnyView(
+          playerContent(itemAudioProperties)
+            .tint(.appGrayTextContrast)
+        )
       } else {
         return AnyView(EmptyView())
       }

apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift

@@ -34,6 +34,9 @@
     }
 
     var playPauseButtonItem: some View {
+      if audioController.playbackError {
+        return AnyView(Color.clear)
+      }
       if let itemID = audioController.itemAudioProperties?.itemID, audioController.isLoadingItem(itemID: itemID) {
         return AnyView(ProgressView())
       } else {
@@ -121,21 +124,25 @@
     func playerContent(_ itemAudioProperties: LinkedItemAudioProperties) -> some View {
       VStack(spacing: 0) {
         HStack(alignment: .center, spacing: 15) {
-          artwork(itemAudioProperties, forDimensions: 50)
+          if audioController.playbackError {
+            Text("There was an error playing back your audio.").foregroundColor(Color.red).font(.footnote)
+            Spacer(minLength: 0)
+          } else {
+            artwork(itemAudioProperties, forDimensions: 50)
 
-          Text(itemAudioProperties.title)
-            .font(Font.system(size: 17, weight: .medium))
-            .fixedSize(horizontal: false, vertical: true)
-            .lineLimit(2)
-            .foregroundColor(.appGrayTextContrast)
-            .frame(maxHeight: 40, alignment: .leading)
+            Text(itemAudioProperties.title)
+              .font(Font.system(size: 17, weight: .medium))
+              .fixedSize(horizontal: false, vertical: true)
+              .lineLimit(2)
+              .foregroundColor(.appGrayTextContrast)
+              .frame(maxHeight: 40, alignment: .leading)
 
-          Spacer(minLength: 0)
-
-          playPauseButtonItem
-            .frame(width: 40, height: 40)
-            .foregroundColor(.themeAudioPlayerGray)
+            Spacer(minLength: 0)
 
+            playPauseButtonItem
+              .frame(width: 40, height: 40)
+              .foregroundColor(.themeAudioPlayerGray)
+          }
           stopButton
             .frame(width: 40, height: 40)
             .foregroundColor(.themeAudioPlayerGray)
@@ -169,14 +176,9 @@
               Spacer(minLength: 0)
               playerContent(itemAudioProperties)
                 .frame(maxHeight: expanded ? 0 : 110)
-              //    .tint(.appGrayTextContrast)
-              // .background(Color.systemBackground)
             }
           }
         }
-      }.alert("There was an error playing back your audio.",
-              isPresented: $audioController.playbackError) {
-        Button(LocalText.dismissButton, role: .none) {}
       }
     }
   }

apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift

@@ -126,7 +126,7 @@ struct WebReaderContainerView: View {
 
   #if os(iOS)
     var audioNavbarItem: some View {
-      if audioController.isLoadingItem(itemID: item.unwrappedID) {
+      if !audioController.playbackError, audioController.isLoadingItem(itemID: item.unwrappedID) {
         return AnyView(ProgressView()
           .padding(.horizontal))
       } else {
@@ -157,7 +157,7 @@ struct WebReaderContainerView: View {
     }
 
     var textToSpeechButtonImage: some View {
-      if audioController.state == .stopped || audioController.itemAudioProperties?.itemID != self.item.id {
+      if audioController.playbackError || audioController.state == .stopped || audioController.itemAudioProperties?.itemID != self.item.id {
         return AnyView(Image.headphones)
       }
       let name = audioController.isPlayingItem(itemID: item.unwrappedID) ? "pause.circle" : "play.circle"

apple/OmnivoreKit/Sources/Models/AppEnvironment.swift

@@ -103,7 +103,7 @@ public extension AppEnvironment {
     case .prod:
       return URL(string: prodTtsURL)!
     case .test, .local:
-      return URL(string: "http://localhost:4000")!
+      return URL(string: "http://localhost:8080")!
     case .custom:
       guard
         let str = UserDefaults.standard.string(forKey: AppEnvironmentUserDefaultKey.ttsBaseURL.rawValue),

apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift

@@ -56,7 +56,7 @@
     var durations: [Double]?
     var lastReadUpdate = 0.0
 
-    var samplePlayer: AVAudioPlayer?
+    var samplePlayer: AVPlayer?
 
     public init(dataService: DataService) {
       self.dataService = dataService
@@ -134,6 +134,12 @@
     public func stopWithError() {
       pause()
       playbackError = true
+
+      timer?.invalidate()
+      timer = nil
+      if let player = player {
+        player.removeAllItems()
+      }
     }
 
     public func generateVoiceList() -> [VoiceItem] {
@@ -475,36 +481,36 @@
       return "en-US-CoraNeural"
     }
 
+    func previewVoiceURL(_ voice: String) -> URL? {
+      URL(string: "https://storage.googleapis.com/omnivore_preview_bucket/tts-voice-previews/\(voice).mp3")
+    }
+
     public func playVoiceSample(voice: String) {
-      do {
-        pause()
+      pause()
 
-        if let url = Bundle(url: UtilsPackage.bundleURL)?.url(forResource: voice, withExtension: "mp3") {
-          try AVAudioSession.sharedInstance().setCategory(.playback, mode: .default, options: [])
-
-          samplePlayer = try AVAudioPlayer(contentsOf: url, fileTypeHint: AVFileType.mp3.rawValue)
-          if !(samplePlayer?.play() ?? false) {
-            throw BasicError.message(messageText: "Unable to playback audio")
-          }
-        } else {
-          NSNotification.operationFailed(message: "Error playing voice sample.")
+      if let url = previewVoiceURL(voice) {
+        samplePlayer = AVPlayer(playerItem: AVPlayerItem(url: url))
+        if let samplePlayer = samplePlayer {
+          samplePlayer.play()
         }
-      } catch {
-        print("ERROR", error)
+      } else {
         NSNotification.operationFailed(message: "Error playing voice sample.")
       }
     }
 
     public func isPlayingSample(voice: String) -> Bool {
-      if let samplePlayer = self.samplePlayer, let url = Bundle(url: UtilsPackage.bundleURL)?.url(forResource: voice, withExtension: "mp3") {
-        return samplePlayer.url == url && samplePlayer.isPlaying
+      if let samplePlayer = self.samplePlayer, let url = previewVoiceURL(voice) {
+        if let urlAsset = samplePlayer.currentItem?.asset as? AVURLAsset {
+          return urlAsset.url == url && samplePlayer.timeControlStatus == .playing
+        }
+        return false
       }
       return false
     }
 
     public func stopVoiceSample() {
       if let samplePlayer = self.samplePlayer {
-        samplePlayer.stop()
+        samplePlayer.pause()
         self.samplePlayer = nil
       }
     }
@@ -576,8 +582,7 @@
               let startOffset = index < document.utterances.count ? offset : 0.0
               self.startStreamingAudio(itemID: itemID, document: document, atIndex: startIndex, andOffset: startOffset)
             } else {
-              print("unable to load speech document")
-              // TODO: Post error to SnackBar
+              self.stopWithError()
             }
           }
         }

apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift

@@ -17,6 +17,7 @@ struct UtteranceRequest: Codable {
   let language: String
   let rate: String
   let isUltraRealisticVoice: Bool
+  let isOpenAIVoice: Bool
 }
 
 struct Utterance: Decodable {
@@ -32,7 +33,8 @@ struct Utterance: Decodable {
                                    voice: usedVoice,
                                    language: document.language,
                                    rate: "1.1",
-                                   isUltraRealisticVoice: Voices.isUltraRealisticVoice(usedVoice))
+                                   isUltraRealisticVoice: Voices.isUltraRealisticVoice(usedVoice),
+                                   isOpenAIVoice: Voices.isOpenAIVoice(usedVoice))
     return try JSONEncoder().encode(request)
   }
 }

apple/OmnivoreKit/Sources/Services/AudioSession/Voices.swift

@@ -62,6 +62,10 @@ public enum Voices {
     })
   }
 
+  public static func isOpenAIVoice(_ voiceKey: String) -> Bool {
+    voiceKey.starts(with: "openai-")
+  }
+
   public static let English = VoiceLanguage(key: "en",
                                             name: "English",
                                             defaultVoice: "en-US-ChristopherNeural",
@@ -81,6 +85,10 @@ public enum Voices {
   ]
 
   public static let Pairs = [
+    VoicePair(firstKey: "openai-alloy", secondKey: "openai-echo", firstName: "Alloy", secondName: "Echo", language: "en-US", category: .enUS),
+    VoicePair(firstKey: "openai-fable", secondKey: "openai-onyx", firstName: "Fable", secondName: "Onyx", language: "en-US", category: .enUS),
+    VoicePair(firstKey: "openai-nova", secondKey: "openai-shimmer", firstName: "Nova", secondName: "Shimmer", language: "en-US", category: .enUS),
+
     VoicePair(firstKey: "en-US-JennyNeural", secondKey: "en-US-BrandonNeural", firstName: "Jenny", secondName: "Brandon", language: "en-US", category: .enUS),
     VoicePair(firstKey: "en-US-CoraNeural", secondKey: "en-US-ChristopherNeural", firstName: "Cora", secondName: "Christopher", language: "en-US", category: .enUS),
     VoicePair(firstKey: "en-US-ElizabethNeural", secondKey: "en-US-EricNeural", firstName: "Elizabeth", secondName: "Eric", language: "en-US", category: .enUS),

apple/OmnivoreKit/Sources/Views/FeedItem/GridCard.swift

@@ -74,7 +74,7 @@ public struct GridCard: View {
 
       ZStack(alignment: .bottomLeading) {
         if let imageURL = item.imageURL {
-          AsyncImage(url: imageURL) { phase in
+          CachedAsyncImage(url: imageURL) { phase in
             switch phase {
             case .empty:
               Color.systemBackground

apple/OmnivoreKit/Sources/Views/FeedItem/LibraryFeatureCard.swift

@@ -34,7 +34,7 @@ public struct LibraryFeatureCard: View {
   var imageBox: some View {
     ZStack(alignment: .bottomLeading) {
       if let imageURL = item.imageURL {
-        AsyncImage(url: imageURL) { phase in
+        CachedAsyncImage(url: imageURL) { phase in
           switch phase {
           case .empty:
             Color.systemBackground

apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift

@@ -241,7 +241,7 @@ public struct LibraryItemCard: View {
   var imageBox: some View {
     ZStack(alignment: .bottomLeading) {
       if let imageURL = item.imageURL {
-        AsyncImage(url: imageURL) { phase in
+        CachedAsyncImage(url: imageURL) { phase in
           if let image = phase.image {
             image
               .resizable()

apple/OmnivoreKit/Sources/Views/Images/CachedAsyncImage.swift

@@ -0,0 +1,66 @@
+//
+//  CachedAsyncImage.swift
+//
+//
+//  Created by Jackson Harper on 11/8/23.
+//
+
+// from: https://github.com/pitt500/Pokedex
+
+import SwiftUI
+
+struct CachedAsyncImage<Content>: View where Content: View {
+  private let url: URL
+  private let scale: CGFloat
+  private let transaction: Transaction
+  private let content: (AsyncImagePhase) -> Content
+
+  init(
+    url: URL,
+    scale: CGFloat = 1.0,
+    transaction: Transaction = Transaction(),
+    @ViewBuilder content: @escaping (AsyncImagePhase) -> Content
+  ) {
+    self.url = url
+    self.scale = scale
+    self.transaction = transaction
+    self.content = content
+  }
+
+  var body: some View {
+    if let cached = ImageCache[url] {
+      // _ = print("cached \(url.absoluteString)")
+      content(.success(cached))
+    } else {
+      // _ = print("request \(url.absoluteString)")
+      AsyncImage(
+        url: url,
+        scale: scale,
+        transaction: transaction
+      ) { phase in
+        cacheAndRender(phase: phase)
+      }
+    }
+  }
+
+  func cacheAndRender(phase: AsyncImagePhase) -> some View {
+    if case let .success(image) = phase {
+      ImageCache[url] = image
+    }
+
+    return content(phase)
+  }
+}
+
+private enum ImageCache {
+  private static var cache: [URL: Image] = [:]
+
+  static subscript(url: URL) -> Image? {
+    get {
+      ImageCache.cache[url]
+    }
+    set {
+      ImageCache.cache[url] = newValue
+    }
+  }
+}

packages/text-to-speech/src/azureTextToSpeech.ts

@@ -18,7 +18,7 @@ import {
 
 export class AzureTextToSpeech implements TextToSpeech {
   use(input: TextToSpeechInput): boolean {
-    return !input.isUltraRealisticVoice
+    return !input.isUltraRealisticVoice && !input.isOpenAIVoice
   }
 
   synthesizeTextToSpeech = async (

packages/text-to-speech/src/index.ts

@@ -19,6 +19,7 @@ import {
 } from './textToSpeech'
 import { createClient } from 'redis'
 import { RealisticTextToSpeech } from './realisticTextToSpeech'
+import { OpenAITextToSpeech } from './openaiTextToSpeech'
 
 // explicitly create the return type of RedisClient
 type RedisClient = ReturnType<typeof createClient>
@@ -63,6 +64,7 @@ const MAX_CHARACTER_COUNT = 50000
 const storage = new Storage()
 
 const textToSpeechHandlers = [
+  new OpenAITextToSpeech(),
   new AzureTextToSpeech(),
   new RealisticTextToSpeech(),
 ]

packages/text-to-speech/src/openaiTextToSpeech.ts

@@ -0,0 +1,56 @@
+import {
+  TextToSpeech,
+  TextToSpeechInput,
+  TextToSpeechOutput,
+} from './textToSpeech'
+import axios from 'axios'
+import { stripEmojis } from './htmlToSsml'
+
+const OPEN_AI_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer']
+
+export class OpenAITextToSpeech implements TextToSpeech {
+  synthesizeTextToSpeech = async (
+    input: TextToSpeechInput
+  ): Promise<TextToSpeechOutput> => {
+    const apiKey = process.env.OPENAI_API_KEY
+    const voice = input.voice?.substring('openai-'.length)
+
+    if (!apiKey) {
+      throw new Error('API credentials not set')
+    }
+
+    const HEADERS = {
+      Authorization: `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    }
+
+    const payload = {
+      model: 'tts-1',
+      voice: voice,
+      input: stripEmojis(input.text),
+    }
+
+    const requestUrl = `https://api.openai.com/v1/audio/speech`
+    const response = await axios.post<Buffer>(requestUrl, payload, {
+      headers: HEADERS,
+      responseType: 'arraybuffer',
+    })
+
+    if (response.data.length === 0) {
+      console.log('No payload returned: ', response)
+      throw new Error('No payload returned')
+    }
+
+    return {
+      speechMarks: [],
+      audioData: response.data,
+    }
+  }
+
+  use(input: TextToSpeechInput): boolean {
+    if (input.voice?.startsWith('openai-')) {
+      return true
+    }
+    return false
+  }
+}

packages/text-to-speech/src/textToSpeech.ts

@@ -7,6 +7,7 @@ export interface TextToSpeechInput {
   rate?: string
   secondaryVoice?: string
   audioStream?: NodeJS.ReadWriteStream
+  isOpenAIVoice?: boolean
   isUltraRealisticVoice?: boolean
 }