diff --git a/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift b/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift index d2472344f..f232375c7 100644 --- a/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift @@ -1,123 +1,269 @@ import SwiftUI import Models import Services +import Views +import MarkdownUI +import Utils +@MainActor public class FullScreenDigestViewModel: ObservableObject { @Published var isLoading = false @Published var digest: DigestResult? + @AppStorage(UserDefaultKey.lastVisitedDigestId.rawValue) var lastVisitedDigestId = "" - func load(dataService: DataService) async { - isLoading = true - if digest == nil { - do { - digest = try await dataService.getLatestDigest(timeoutInterval: 10) - } catch { - print("ERROR WITH DIGEST: ", error) - } + func load(dataService: DataService, audioController: AudioController) async { + if let digest = dataService.loadStoredDigest() { + self.digest = digest + } else { + isLoading = true } + do { + if let digest = try await dataService.getLatestDigest(timeoutInterval: 10) { + self.digest = digest + lastVisitedDigestId = digest.id + if let playingDigest = audioController.itemAudioProperties as? DigestAudioItem, playingDigest.digest.id == digest.id { + // Don't think we need to do anything here + } else { + audioController.play(itemAudioProperties: DigestAudioItem(digest: digest)) + } + } + } catch { + print("ERROR WITH DIGEST: ", error) + } + isLoading = false } -} -struct DigestAudioItem: AudioItemProperties { - let audioItemType = Models.AudioItemType.digest - - var itemID = "" - - var title = "TITLE" - - var byline: String? = "byline" - - var imageURL: URL? = nil - - var language: String? - - var startIndex: Int = 0 - var startOffset: Double = 0.0 + func refreshDigest(dataService: DataService) async { + do { + try await dataService.refreshDigest() + } catch { + print("ERROR WITH DIGEST: ", error) + } + } } @available(iOS 17.0, *) @MainActor struct FullScreenDigestView: View { - let viewModel: DigestViewModel = DigestViewModel() + @StateObject var viewModel = FullScreenDigestViewModel() let dataService: DataService let audioController: AudioController @Environment(\.dismiss) private var dismiss - let textBody = "In a significant political turn, the SOTU response faces unexpected collapse, " + - "marking a stark contrast to Trump's latest downturn, alongside an unprecedented " + - "surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " + - "The analysis provides insights into the shifting dynamics of political support and " + - "the potential implications for future electoral strategies. Based on the information " + - "you provided, the video seems to discuss a recent event where former President " + - "Donald Trump made a controversial statement that shocked even his own audience. " + - "The video likely covers Trump's response to the State of the Union (SOTU) address " + - "and how it received negative feedback, possibly leading to a decline in his support " + - "or approval ratings. Additionally, it appears that the video touches upon a surge " + - "in fundraising for President Joe Biden's administration around March 11, 2024." - public init(dataService: DataService, audioController: AudioController) { self.dataService = dataService self.audioController = audioController } - var body: some View { - // ZStack(alignment: Alignment(horizontal: .trailing, vertical: .top)) { - Group { - if viewModel.isLoading { - ProgressView() - } else { - itemBody - .task { - await viewModel.load(dataService: dataService) - }.onAppear { - self.audioController.play(itemAudioProperties: DigestAudioItem()) - } - } - } .navigationTitle("Omnivore digest") - .navigationBarTitleDisplayMode(.inline) + var titleBlock: some View { + HStack { + Text("Omnivore Digest") + .font(Font.system(size: 18, weight: .semibold)) + Image.tabDigestSelected + Spacer() + closeButton + } + .padding(.top, 20) + .padding(.horizontal, 20) + } -// HStack(alignment: .top) { -// Spacer() -// closeButton -// } -// .padding(20) - // } + var createdString: String { + if let createdAt = viewModel.digest?.createdAt, + let date = DateFormatter.formatterISO8601.date(from: createdAt) { + let dateFormatter = DateFormatter() + dateFormatter.dateStyle = .medium + dateFormatter.timeStyle = .medium + dateFormatter.locale = Locale(identifier: "en_US") + + return "Created " + dateFormatter.string(from: date) + } + return "" + } + + var body: some View { + VStack { + titleBlock + + Group { + if viewModel.isLoading { + VStack { + Spacer() + ProgressView() + Spacer() + } + } else { + itemBody + } + } + .edgesIgnoringSafeArea(.bottom) + + }.task { + await viewModel.load(dataService: dataService, audioController: audioController) + } } var closeButton: some View { Button(action: { dismiss() }, label: { - ZStack { - Circle() - .foregroundColor(Color.appGrayText) - .frame(width: 36, height: 36) - .opacity(0.1) - - Image(systemName: "xmark") - .font(.appCallout) - .frame(width: 36, height: 36) - } + Text("Close") + .foregroundColor(Color.blue) }) .buttonStyle(.plain) } + func getChapterData(digest: DigestResult) -> [String:(time: String, start: Int, end: Int)] { + let speed = 1.0 + var chapterData: [String:(time: String, start: Int, end: Int)] = [:] + var currentAudioIndex = 0 + var currentWordCount = 0.0 + + for (index, speechFile) in digest.speechFiles.enumerated() { + let chapter = digest.chapters[index] + let duration = currentWordCount / SpeechDocument.averageWPM / speed * 60.0 + + chapterData[chapter.id] = ( + time: formatTimeInterval(duration) ?? "00:00", + start: Int(currentAudioIndex), + end: currentAudioIndex + Int(speechFile.utterances.count) + ) + currentAudioIndex += Int(speechFile.utterances.count) + currentWordCount += chapter.wordCount + } + return chapterData + } + + func formatTimeInterval(_ time: TimeInterval) -> String? { + let componentFormatter = DateComponentsFormatter() + componentFormatter.unitsStyle = .positional + componentFormatter.allowedUnits = time >= 3600 ? [.second, .minute, .hour] : [.second, .minute] + componentFormatter.zeroFormattingBehavior = .pad + return componentFormatter.string(from: time) + } + @available(iOS 17.0, *) var itemBody: some View { VStack { - ScrollView(.vertical) { - VStack(spacing: 20) { - Text("SOTU response collapses, Trump hits new low, Biden fundraising explodes 3/11/24 TDPS Podcast") - .font(.title) - Text(textBody) - .font(.body) + ScrollView { + VStack(alignment: .leading, spacing: 20) { + HStack { + Image.coloredSmallOmnivoreLogo + .resizable() + .frame(width: 20, height: 20) + Text("Omnivore.app") + .font(Font.system(size: 14)) + .foregroundColor(Color.themeLibraryItemSubtle) + Spacer() + } + if let digest = viewModel.digest { + Text(digest.title) + .font(Font.system(size: 17, weight: .semibold)) + .lineSpacing(5) + .lineLimit(3) + Text(createdString) + .font(Font.system(size: 12)) + .foregroundColor(Color(hex: "#898989")) + .lineLimit(1) + Text(digest.description) + .font(Font.system(size: 14)) + .lineSpacing(/*@START_MENU_TOKEN@*/10.0/*@END_MENU_TOKEN@*/) + .foregroundColor(Color.themeLibraryItemSubtle) + .lineLimit(6) + } else { + Text("We're building you a new digest") + .font(Font.system(size: 17, weight: .semibold)) + .lineLimit(3) + ProgressView() + } } - } - // .scrollTargetBehavior(.paging) - // .ignoresSafeArea() - MiniPlayerViewer() + .padding(15) + .background(Color.themeLabelBackground.opacity(0.6)) + .cornerRadius(5) + + if let digest = viewModel.digest { + VStack(alignment: .leading, spacing: 10) { + Text("Chapters") + .font(Font.system(size: 17, weight: .semibold)) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(0) + let chapterData = getChapterData(digest: digest) + ForEach(digest.chapters, id: \.id) { chapter in + if let startTime = chapterData[chapter.id]?.time, let skipIndex = chapterData[chapter.id]?.start { + let currentChapter = audioController.currentAudioIndex >= (chapterData[chapter.id]?.start ?? 0) && + audioController.currentAudioIndex < (chapterData[chapter.id]?.end ?? 0) + ChapterView( + startTime: startTime, + skipIndex: skipIndex, + chapter: chapter + ) + .onTapGesture { + audioController.seek(toIdx: skipIndex) + } + .background( + currentChapter ? Color.themeLabelBackground.opacity(0.6) : Color.clear + ) + .cornerRadius(5) + } + } + } + .padding(.top, 20) + } + + if let digest = viewModel.digest { + Text("Transcript") + .font(Font.system(size: 17, weight: .semibold)) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.top, 20) + + VStack { + Markdown(digest.content) + .foregroundColor(Color.appGrayTextContrast) + } + .padding(15) + .background(Color.themeLabelBackground.opacity(0.6)) + .cornerRadius(5) + } + + Spacer(minLength: 60) + + if viewModel.digest != nil { + Text("Rate today's digest") + .font(Font.system(size: 17, weight: .semibold)) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.bottom, 15) + .padding(.horizontal, 15) + + RatingWidget() + Spacer(minLength: 60) + } + + VStack(alignment: .leading, spacing: 20) { + Text("If you didn't like today's digest or would like another one you can create another one. The process takes a few minutes") + Button(action: { + Task { + await viewModel.refreshDigest(dataService: dataService) + } + }, label: { + Text("Create new digest") + .font(Font.system(size: 13, weight: .medium)) + .padding(.horizontal, 8) + .padding(.vertical, 5) + .tint(Color.blue) + .background(Color.themeLabelBackground) + .cornerRadius(5) + }) + } + .padding(15) + .background(Color.themeLabelBackground.opacity(0.6)) + .cornerRadius(5) + + }.contentMargins(10, for: .scrollContent) + + Spacer() + + MiniPlayerViewer(showStopButton: false) .padding(.top, 10) .padding(.bottom, 40) .background(Color.themeTabBarColor) @@ -128,6 +274,50 @@ struct FullScreenDigestView: View { } } +struct ChapterView: View { + let startTime: String + let skipIndex: Int + let chapter: DigestChapter + + var body: some View { + HStack(spacing: 15) { + if let thumbnail = chapter.thumbnail, let thumbnailURL = URL(string: thumbnail) { + AsyncImage(url: thumbnailURL) { image in + image + .resizable() + .aspectRatio(contentMode: .fill) + .frame(width: 90, height: 50) + .clipped() + } placeholder: { + Rectangle() + .foregroundColor(.gray) + .frame(width: 90, height: 50) + } + .cornerRadius(8) + } else { + Rectangle() + .foregroundColor(.gray) + .frame(width: 90, height: 50) + .cornerRadius(8) + } + VStack(alignment: .leading) { + (Text(startTime) + .foregroundColor(.blue) + .font(.caption) + + + + Text(" - " + chapter.title) + .foregroundColor(.primary) + .font(.caption)) + .lineLimit(2) + } + Spacer() + } + .padding(.leading, 4) + .padding(.vertical, 15) + } +} + @MainActor public class PreviewItemViewModel: ObservableObject { let dataService: DataService @@ -145,24 +335,6 @@ public class PreviewItemViewModel: ObservableObject { } func loadResult() async { -// isLoading = true -// let taskId = try? await dataService.createAITask( -// extraText: extraText, -// libraryItemId: item?.id ?? "", -// promptName: "summarize-001" -// ) -// -// if let taskId = taskId { -// do { -// let fetchedText = try await dataService.pollAITask(jobId: taskId, timeoutInterval: 30) -// resultText = fetchedText -// } catch { -// print("ERROR WITH RESULT TEXT: ", error) -// } -// } else { -// print("NO TASK ID: ", taskId) -// } -// isLoading = false } } @@ -200,7 +372,7 @@ struct PreviewItemView: View { .foregroundColor(Color(hex: "898989")) .frame(maxWidth: .infinity, alignment: .topLeading) - Color(hex: "2A2A2A") + Color.themeLabelBackground .frame(height: 1) .frame(maxWidth: .infinity, alignment: .center) .padding(.vertical, 20) @@ -296,7 +468,7 @@ struct RatingWidget: View { } } .padding() - .background(Color(hex: "313131")) + .background(Color.themeLabelBackground.opacity(0.6)) .cornerRadius(8) // .shadow(radius: 3) } diff --git a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift index fb0c31b20..c3973c2ac 100644 --- a/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift +++ b/apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift @@ -8,12 +8,16 @@ import Views public struct MiniPlayerViewer: View { + var showStopButton = true @EnvironmentObject var audioController: AudioController @Environment(\.colorScheme) private var colorScheme: ColorScheme @State var expanded = true var playPauseButtonImage: String { +#if targetEnvironment(simulator) + return "play.circle" +#endif switch audioController.state { case .playing: return "pause.circle" @@ -27,6 +31,17 @@ } var playPauseButtonItem: some View { +#if targetEnvironment(simulator) + return AnyView(Button( + action: {}, + label: { + Image(systemName: playPauseButtonImage) + .resizable(resizingMode: Image.ResizingMode.stretch) + .aspectRatio(contentMode: .fit) + .font(Font.title.weight(.light)) + } + ).buttonStyle(.plain)) +#endif if audioController.playbackError { return AnyView(Color.clear) } @@ -138,9 +153,11 @@ .frame(width: 40, height: 40) .foregroundColor(.themeAudioPlayerGray) } - stopButton - .frame(width: 40, height: 40) - .foregroundColor(.themeAudioPlayerGray) + if showStopButton { + stopButton + .frame(width: 40, height: 40) + .foregroundColor(.themeAudioPlayerGray) + } } .padding(.vertical, 5) .padding(.horizontal, 15) diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift index 3b22ac24e..430b16d26 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift @@ -181,7 +181,7 @@ struct AnimatingCellHeight: AnimatableModifier { // swiftlint:disable file_length #if os(iOS) - private let enableGrid = UIDevice.isIPad || FeatureFlag.enableGridCardsOnPhone + private let enableGrid = UIDevice.isIPad @MainActor struct HomeFeedContainerView: View { @@ -204,9 +204,6 @@ struct AnimatingCellHeight: AnimatableModifier { @ObservedObject var viewModel: HomeFeedViewModel @State private var selection = Set() - @AppStorage("LibraryList::digestEnabled") var digestEnabled = false - @AppStorage("LibraryList::hasCheckedForDigestFeature") var hasCheckedForDigestFeature = false - init(viewModel: HomeFeedViewModel, isEditMode: Binding) { _viewModel = ObservedObject(wrappedValue: viewModel) _isEditMode = isEditMode @@ -274,7 +271,11 @@ struct AnimatingCellHeight: AnimatableModifier { .padding(.bottom, 20) .background(Color.themeTabBarColor) .onTapGesture { - showExpandedAudioPlayer = true + if audioController.itemAudioProperties?.audioItemType == .digest { + showLibraryDigest = true + } else { + showExpandedAudioPlayer = true + } } } } @@ -351,20 +352,6 @@ struct AnimatingCellHeight: AnimatableModifier { viewModel.stopUsingFollowingPrimer = true } } - .task { - do { - if let viewer = try await dataService.fetchViewer() { - digestEnabled = viewer.digestEnabled ?? false - if !hasCheckedForDigestFeature { - hasCheckedForDigestFeature = true - // selectedTab = "digest" - } - } - } catch { - print("ERROR FETCHING VIEWER: ", error) - print("") - } - } .environment(\.editMode, self.$isEditMode) .navigationBarTitleDisplayMode(.inline) } @@ -409,10 +396,10 @@ struct AnimatingCellHeight: AnimatableModifier { if isEditMode == .active { Button(action: { isEditMode = .inactive }, label: { Text("Cancel") }) } else { - if #available(iOS 17.0, *) { + if #available(iOS 17.0, *), dataService.featureFlags.digestEnabled { Button( action: { showLibraryDigest = true }, - label: { Image.tabDigestSelected } + label: { viewModel.digestIsUnread ? Image.tabDigestSelected : Image.tabDigest } ) .buttonStyle(.plain) .padding(.trailing, 4) diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift index 815e85a54..35aa4992f 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift @@ -38,6 +38,8 @@ enum LoadingBarStyle { @Published var selectedLabels = [LinkedItemLabel]() @Published var negatedLabels = [LinkedItemLabel]() @Published var appliedSort = LinkedItemSort.newest.rawValue + + @Published var digestIsUnread = false @State var lastMoreFetched: Date? @State var lastFiltersFetched: Date? @@ -47,6 +49,7 @@ enum LoadingBarStyle { @AppStorage(UserDefaultKey.hideFeatureSection.rawValue) var hideFeatureSection = false @AppStorage(UserDefaultKey.stopUsingFollowingPrimer.rawValue) var stopUsingFollowingPrimer = false @AppStorage("LibraryTabView::hideFollowingTab") var hideFollowingTab = false + @AppStorage(UserDefaultKey.lastVisitedDigestId.rawValue) var lastVisitedDigestId = "" @AppStorage(UserDefaultKey.lastSelectedFeaturedItemFilter.rawValue) var featureFilter = FeaturedItemFilter.continueReading.rawValue @@ -395,4 +398,14 @@ enum LoadingBarStyle { isEmptyingTrash = false } } + + func checkForDigestUpdate(dataService: DataService) async { + do { + if let result = try? await dataService.getLatestDigest(timeoutInterval: 2) { + if result.id != lastVisitedDigestId { + digestIsUnread = true + } + } + } + } } diff --git a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift index e98786b5a..06908b2cd 100644 --- a/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift @@ -15,6 +15,7 @@ struct LibraryTabView: View { @AppStorage(UserDefaultKey.lastSelectedTabItem.rawValue) var selectedTab = "inbox" @State var isEditMode: EditMode = .inactive + @State var showLibraryDigest = false @State var showExpandedAudioPlayer = false @State var presentPushContainer = true @State var pushLinkRequest: String? @@ -69,27 +70,13 @@ struct LibraryTabView: View { @State var operationStatus: OperationStatus = .none @State var operationMessage: String? - @State var digestEnabled = false - - var showDigest: Bool { - if digestEnabled, #available(iOS 17.0, *) { - return true - } - return false - } - var displayTabs: [String] { var res = [String]() if !hideFollowingTab { res.append("following") } - if showDigest { - res.append("digest") - } res.append("inbox") - if !showDigest { - res.append("profile") - } + res.append("profile") return res } @@ -133,34 +120,26 @@ struct LibraryTabView: View { }.tag("following") } - if showDigest, #available(iOS 17.0, *) { - NavigationView { - DigestView(dataService: dataService) - .navigationBarTitleDisplayMode(.inline) - .navigationViewStyle(.stack) - }.tag("digest") - NavigationView { - HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode) - .navigationBarTitleDisplayMode(.inline) - .navigationViewStyle(.stack) - }.tag("inbox") - } else { - NavigationView { - HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode) - .navigationBarTitleDisplayMode(.inline) - .navigationViewStyle(.stack) - }.tag("inbox") - NavigationView { - ProfileView() - .navigationViewStyle(.stack) - }.tag("profile") - } + NavigationView { + HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode) + .navigationBarTitleDisplayMode(.inline) + .navigationViewStyle(.stack) + }.tag("inbox") + NavigationView { + ProfileView() + .navigationViewStyle(.stack) + }.tag("profile") } + if audioController.itemAudioProperties != nil { MiniPlayerViewer() .onTapGesture { - showExpandedAudioPlayer = true + if audioController.itemAudioProperties?.audioItemType == .digest { + showLibraryDigest = true + } else { + showExpandedAudioPlayer = true + } } .padding(0) Color(hex: "#3D3D3D") @@ -193,6 +172,15 @@ struct LibraryTabView: View { } ) } + .fullScreenCover(isPresented: $showLibraryDigest) { + if #available(iOS 17.0, *) { + NavigationView { + FullScreenDigestView(dataService: dataService, audioController: audioController) + } + } else { + Text("Sorry digest is only available on iOS 17 and above") + } + } .navigationBarHidden(true) .onReceive(NSNotification.performSyncPublisher) { _ in Task { diff --git a/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift b/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift index 46389bfbe..4b6fd3fdd 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Profile/TextToSpeechVoiceSelectionView.swift @@ -23,7 +23,7 @@ var body: some View { Group { Form { - if FeatureFlag.enableUltraRealisticVoices, language.key == "en" { + if language.key == "en" { if viewModel.waitingForRealisticVoices { HStack { Text(LocalText.texttospeechBetaSignupInProcess) diff --git a/apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift b/apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift index ceaa39cd0..18cc21f3d 100644 --- a/apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift +++ b/apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift @@ -48,7 +48,7 @@ func removeLibraryItemAction(dataService: DataService, objectID: NSManagedObject } func archiveLibraryItemAction(dataService: DataService, objectID: NSManagedObjectID, archived: Bool) { - var localPdf: String? = nil + var localPdf: String? dataService.viewContext.performAndWait { if let item = dataService.viewContext.object(with: objectID) as? Models.LibraryItem { item.isArchived = archived diff --git a/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift b/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift index 7260e6bab..3822faa1e 100644 --- a/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift @@ -37,6 +37,8 @@ public struct RootView: View { Services.scheduleBackgroundFetch() #endif } + }.task { + await viewModel.services.dataService.tryUpdateFeatureFlags() } } } diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/ExplainView.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/ExplainView.swift new file mode 100644 index 000000000..12131ff3b --- /dev/null +++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/ExplainView.swift @@ -0,0 +1,57 @@ +// swiftlint:disable line_length +import Foundation +import Models +import SwiftUI +import Views +import WebKit +import Services + +@MainActor public final class ExplainViewModel: ObservableObject { + @Published var isLoading = true + @Published var explanation = "" + + func load(dataService: DataService, text: String, libraryItemId: String) async { + isLoading = true + + do { + + explanation = try await dataService.explain(text: text, libraryItemId: libraryItemId) + } catch { + print("ERROR: ", error) + explanation = "There was an error generating your explanation" + } + + isLoading = false + } +} + +@MainActor +struct ExplainView: View { + let dataService: DataService + + let text: String + let item: Models.LibraryItem + + @StateObject var viewModel = ExplainViewModel() + + init(dataService: DataService, text: String, item: Models.LibraryItem) { + self.text = text + self.item = item + self.dataService = dataService + } + + var body: some View { + if viewModel.isLoading { + ProgressView() + .task { + await viewModel.load(dataService: dataService, text: text, libraryItemId: item.unwrappedID) + } + } else { + Text(viewModel.explanation) + .font(Font.system(size: 19)) + .lineSpacing(12) + .padding(20) + .frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading) + } + } +} diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/HighlightViewer.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/HighlightViewer.swift deleted file mode 100644 index ef90a7f3f..000000000 --- a/apple/OmnivoreKit/Sources/App/Views/WebReader/HighlightViewer.swift +++ /dev/null @@ -1,93 +0,0 @@ -import Models -import SwiftUI -import Utils -import Views -import WebKit - -#if os(iOS) - struct HighlightViewer: PlatformViewRepresentable { - let highlightData: HighlightData - - func makeCoordinator() -> WebReaderCoordinator { - WebReaderCoordinator() - } - - private func makePlatformView(context: Context) -> WKWebView { - let webView = WebViewManager.shared() - let contentController = WKUserContentController() - - webView.navigationDelegate = context.coordinator - webView.configuration.userContentController = contentController - webView.configuration.userContentController.removeAllScriptMessageHandlers() - - #if os(iOS) - webView.isOpaque = false - webView.backgroundColor = .clear - webView.scrollView.delegate = context.coordinator - webView.scrollView.contentInset.top = readerViewNavBarHeight - webView.scrollView.verticalScrollIndicatorInsets.top = readerViewNavBarHeight - webView.configuration.userContentController.add(webView, name: "viewerAction") - #else - webView.setValue(false, forKey: "drawsBackground") - #endif - - for action in WebViewAction.allCases { - webView.configuration.userContentController.add(context.coordinator, name: action.rawValue) - } - - webView.configuration.userContentController.addScriptMessageHandler( - context.coordinator, contentWorld: .page, name: "articleAction" - ) - - loadContent(webView: webView) - - return webView - } - - private func updatePlatformView(_: WKWebView, context _: Context) { - // If the webview had been terminated `needsReload` will have been set to true - // Or if the articleContent value has changed then it's id will be different from the coordinator's -// if context.coordinator.needsReload { -// loadContent(webView: webView) -// context.coordinator.needsReload = false -// return -// } - } - - private func loadContent(webView: WKWebView) { - // swiftlint:disable line_length - let themeKey = ThemeManager.currentThemeName - let content = """ - - - - - - - - -
-
- \(highlightData.highlightHTML) -
- - - """ - // swiftlint:enable line_length - - webView.loadHTMLString(content, baseURL: ViewsPackage.resourceURL) - } - } - - extension HighlightViewer { - func makeUIView(context: Context) -> WKWebView { - makePlatformView(context: context) - } - - func updateUIView(_ webView: WKWebView, context: Context) { - updatePlatformView(webView, context: context) - } - } -#endif diff --git a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift index 93491b275..e1b6e5653 100644 --- a/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift +++ b/apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift @@ -27,6 +27,7 @@ struct WebReaderContainerView: View { @State var annotationSaveTransactionID: UUID? @State var showNavBarActionID: UUID? @State var showExpandedAudioPlayer = false + @State var showLibraryDigest = false @State var shareActionID: UUID? @State var annotation = String() @State private var bottomBarOpacity = 0.0 @@ -327,6 +328,9 @@ struct WebReaderContainerView: View { .formSheet(isPresented: $showPreferencesFormsheet, modalSize: CGSize(width: 400, height: 475)) { webPreferencesPopoverView } + .formSheet(isPresented: $showExplainSheet, modalSize: CGSize(width: 400, height: 475)) { + explainView + } #endif #if os(macOS) @@ -372,6 +376,10 @@ struct WebReaderContainerView: View { } #endif + var explainView: some View { + ExplainView(dataService: dataService, text: viewModel.explainText ?? "Nothing to explain", item: item) + } + var body: some View { ZStack { if let articleContent = viewModel.articleContent { @@ -392,7 +400,7 @@ struct WebReaderContainerView: View { #endif }, tapHandler: tapHandler, - explainHandler: explainHandler, + explainHandler: dataService.featureFlags.explainEnabled ? explainHandler : nil, scrollPercentHandler: scrollPercentHandler, webViewActionHandler: webViewActionHandler, navBarVisibilityUpdater: { visible in @@ -476,6 +484,15 @@ struct WebReaderContainerView: View { showExpandedAudioPlayer = false }) } + .fullScreenCover(isPresented: $showLibraryDigest) { + if #available(iOS 17.0, *) { + NavigationView { + FullScreenDigestView(dataService: dataService, audioController: audioController) + } + } else { + Text("Sorry digest is only available on iOS 17 and above") + } + } #endif .alert(errorAlertMessage ?? LocalText.readerError, isPresented: $showErrorAlertMessage) { Button(LocalText.genericOk, role: .cancel, action: { @@ -621,7 +638,11 @@ struct WebReaderContainerView: View { .padding(.bottom, showBottomBar ? 10 : 40) .background(Color.themeTabBarColor) .onTapGesture { - showExpandedAudioPlayer = true + if audioController.itemAudioProperties?.audioItemType == .digest { + showLibraryDigest = true + } else { + showExpandedAudioPlayer = true + } } } if showBottomBar { diff --git a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents index 57cf07b2e..e61ca93a9 100644 --- a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents +++ b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents @@ -137,7 +137,6 @@ - diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift b/apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift index 2bccfc724..bf5b6e5a0 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift @@ -1,3 +1,4 @@ +import SwiftUI public struct FeatureInternal { public let name: String diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift index 313265e66..e91017126 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift @@ -1,3 +1,5 @@ +// swiftlint:disable file_length type_body_length + #if os(iOS) import AVFoundation @@ -27,27 +29,55 @@ case high } +public struct DigestAudioItem: AudioItemProperties { + public let audioItemType = Models.AudioItemType.digest + public let digest: DigestResult + public let itemID: String + public let title: String + public var byline: String? + public var imageURL: URL? + public var language: String? + public var startIndex: Int = 0 + public var startOffset: Double = 0.0 + + public init(digest: DigestResult) { + self.digest = digest + self.itemID = digest.id + self.title = digest.title + self.startIndex = 0 + self.startOffset = 0 + + self.imageURL = nil + + if let first = digest.speechFiles.first { + self.language = first.language + self.byline = digest.byline + } + } +} + // swiftlint:disable all + @MainActor public class AudioController: NSObject, ObservableObject, AVAudioPlayerDelegate { @Published public var state: AudioControllerState = .stopped @Published public var currentAudioIndex: Int = 0 @Published public var readText: String = "" @Published public var unreadText: String = "" @Published public var itemAudioProperties: AudioItemProperties? - + @Published public var timeElapsed: TimeInterval = 0 @Published public var duration: TimeInterval = 0 @Published public var timeElapsedString: String? @Published public var durationString: String? @Published public var voiceList: [VoiceItem]? @Published public var realisticVoiceList: [VoiceItem]? - + @Published public var textItems: [String]? - + @Published public var playbackError: Bool = false - + let dataService: DataService - + var timer: Timer? var player: AVQueuePlayer? var observer: Any? @@ -55,30 +85,30 @@ var synthesizer: SpeechSynthesizer? var durations: [Double]? var lastReadUpdate = 0.0 - + var samplePlayer: AVPlayer? - + public init(dataService: DataService) { self.dataService = dataService - + super.init() self.voiceList = generateVoiceList() self.realisticVoiceList = generateRealisticVoiceList() self._currentLanguage = defaultLanguage } - + deinit { player = nil observer = nil } - + public func play(itemAudioProperties: AudioItemProperties) { stop() - + playbackError = false self.itemAudioProperties = itemAudioProperties startAudio(atIndex: itemAudioProperties.startIndex, andOffset: itemAudioProperties.startOffset) - + EventTracker.track( .audioSessionStart( linkID: itemAudioProperties.itemID, @@ -88,7 +118,7 @@ ) ) } - + public var offsets: [Double]? { if let durations = durations { var currentSum = 0.0 @@ -99,55 +129,55 @@ } return nil } - + public func stop() { let stoppedId = itemAudioProperties?.itemID let stoppedTimeElapsed = timeElapsed - + savePositionInfo(force: true) - + player?.pause() timer?.invalidate() - + clearNowPlayingInfo() - + player?.replaceCurrentItem(with: nil) player?.removeAllItems() - + document = nil textItems = nil - + timer = nil player = nil observer = nil synthesizer = nil lastReadUpdate = 0 - + itemAudioProperties = nil state = .stopped timeElapsed = 0 duration = 1 durations = nil currentAudioIndex = 0 - + if let stoppedId = stoppedId { EventTracker.track( .audioSessionEnd(linkID: stoppedId, timeElapsed: stoppedTimeElapsed) ) } } - + public func stopWithError() { pause() playbackError = true - + timer?.invalidate() timer = nil if let player = player { player.removeAllItems() } } - + public func generateVoiceList() -> [VoiceItem] { Voices.Pairs.flatMap { voicePair in [ @@ -156,7 +186,7 @@ ] }.sorted { $0.name.lowercased() < $1.name.lowercased() } } - + public func generateRealisticVoiceList() -> [VoiceItem] { Voices.UltraPairs.flatMap { voicePair in [ @@ -165,14 +195,14 @@ ] }.sorted { $0.name.lowercased() < $1.name.lowercased() } } - + public func preload(itemIDs: [String], retryCount _: Int = 0) async -> Bool { for itemID in itemIDs { _ = try? await downloadSpeechFile(itemID: itemID, priority: .low) } return false } - + public func downloadForOffline(itemID: String) async -> Bool { if let document = try? await getSpeechFile(itemID: itemID, priority: .low) { let synthesizer = SpeechSynthesizer(appEnvironment: dataService.appEnvironment, networker: dataService.networker, document: document, speechAuthHeader: speechAuthHeader) @@ -188,7 +218,7 @@ } return false } - + public static func removeAudioFiles(itemID: String) { do { let audioDirectory = pathForAudioDirectory(itemID: itemID) @@ -199,7 +229,7 @@ print("Error removing audio files", error) } } - + public var scrubState: PlayerScrubState = .reset { didSet { switch scrubState { @@ -212,34 +242,34 @@ } } } - + func updateDuration(forItem item: SpeechItem, newDuration: TimeInterval) { if let durations = self.durations, item.audioIdx < durations.count { self.durations?[item.audioIdx] = (newDuration / playbackRate) } } - + public func seek(toUtterance: Int) { player?.pause() - + player?.removeAllItems() synthesizeFrom(start: toUtterance, playWhenReady: state == .playing, atOffset: 0.0) scrubState = .reset fireTimer() } - + public func seek(to: TimeInterval) { let position = max(0, to) - + // Always reset this state when seeking so we trigger a re-saving of positional info lastReadUpdate = 0 - + // If we are in reachedEnd state, and seek back, we need to move to // paused state if to < duration, state == .reachedEnd { state = .paused } - + // First find the item that this interval is within // Not the most effecient, but these lists should be less than 500 items var sum = 0.0 @@ -251,12 +281,12 @@ } sum += duration } - + if let foundIdx = foundIdx { // Now figure out how far into this segment we need to seek to let before = durationBefore(playerIndex: foundIdx) let remainder = position - before - + // if the foundIdx happens to be the current item, we just set the position if let playerItem = player?.currentItem as? SpeechPlayerItem { if playerItem.speechItem.audioIdx == foundIdx { @@ -266,7 +296,7 @@ return } } - + // Move the playback to the found index, we also seek by the remainder amount // before moving we pause the player so playback doesnt jump to a previous spot player?.pause() @@ -281,17 +311,41 @@ synthesizeFrom(start: durations.count - 1, playWhenReady: state == .playing, atOffset: last) } } - + scrubState = .reset fireTimer() } - + + public func seek(toIdx: Int) { + let before = durationBefore(playerIndex: toIdx) + let remainder = 0.0 + + // if the foundIdx happens to be the current item, we just set the position + if let playerItem = player?.currentItem as? SpeechPlayerItem { + if playerItem.speechItem.audioIdx == toIdx { + playerItem.seek(to: CMTimeMakeWithSeconds(remainder, preferredTimescale: 600), completionHandler: nil) + scrubState = .reset + fireTimer() + return + } + } + + // Move the playback to the found index, we also seek by the remainder amount + // before moving we pause the player so playback doesnt jump to a previous spot + player?.pause() + player?.removeAllItems() + synthesizeFrom(start: toIdx, playWhenReady: state == .playing, atOffset: remainder) + + scrubState = .reset + fireTimer() + } + @AppStorage(UserDefaultKey.textToSpeechDefaultLanguage.rawValue) public var defaultLanguage = "en" { didSet { currentLanguage = defaultLanguage } } - + @AppStorage(UserDefaultKey.textToSpeechPlaybackRate.rawValue) public var playbackRate = 1.0 { didSet { updateDurations(oldPlayback: oldValue, newPlayback: playbackRate) @@ -299,25 +353,25 @@ fireTimer() } } - + @AppStorage(UserDefaultKey.textToSpeechPreloadEnabled.rawValue) public var preloadEnabled = false - + @AppStorage(UserDefaultKey.textToSpeechUseUltraRealisticVoices.rawValue) public var useUltraRealisticVoices = false - + @AppStorage(UserDefaultKey.textToSpeechUltraRealisticFeatureKey.rawValue) public var ultraRealisticFeatureKey: String = "" @AppStorage(UserDefaultKey.textToSpeechUltraRealisticFeatureRequested.rawValue) public var ultraRealisticFeatureRequested: Bool = false - + var speechAuthHeader: String? { if Voices.isUltraRealisticVoice(currentVoice), !ultraRealisticFeatureKey.isEmpty { return ultraRealisticFeatureKey } return nil } - + public var currentVoiceLanguage: VoiceLanguage { Voices.Languages.first(where: { $0.key == currentLanguage }) ?? Voices.English } - + private var _currentLanguage: String? public var currentLanguage: String { get { @@ -331,30 +385,30 @@ } set { _currentLanguage = newValue - + let newVoice = getPreferredVoice(forLanguage: newValue) currentVoice = newVoice } } - + private var _currentVoice: String? public var currentVoice: String { get { if let currentVoice = _currentVoice { return currentVoice } - + if let currentVoice = UserDefaults.standard.string(forKey: "\(currentLanguage)-\(UserDefaultKey.textToSpeechPreferredVoice.rawValue)") { return currentVoice } - + return currentVoiceLanguage.defaultVoice } set { _currentVoice = newValue voiceList = generateVoiceList() realisticVoiceList = generateRealisticVoiceList() - + var currentIdx = 0 var currentOffset = 0.0 if let player = self.player, let item = self.player?.currentItem as? SpeechPlayerItem { @@ -363,11 +417,11 @@ } player?.removeAllItems() playbackError = false - + downloadAndPlayFrom(currentIdx, currentOffset) } } - + public var currentVoicePair: VoicePair? { let voice = currentVoice if Voices.isUltraRealisticVoice(currentVoice) { @@ -378,14 +432,14 @@ } return Voices.Pairs.first(where: { $0.firstKey == voice || $0.secondKey == voice }) } - + struct TextNode: Codable { let to: String let from: String let heading: String let body: String } - + func setTextItems() { if let document = self.document { textItems = document.utterances.map { utterance in @@ -400,16 +454,16 @@ textItems = nil } } - + func updateReadText() { if let item = player?.currentItem as? SpeechPlayerItem, let speechMarks = item.speechMarks { var currentItemOffset = 0 - for i in 0 ..< speechMarks.count { - if speechMarks[i].time ?? 0 < 0 { + for idx in 0 ..< speechMarks.count { + if speechMarks[idx].time ?? 0 < 0 { continue } - if (speechMarks[i].time ?? 0.0) > CMTimeGetSeconds(item.currentTime()) * 1000 { - currentItemOffset = speechMarks[i].start ?? 0 + if (speechMarks[idx].time ?? 0.0) > CMTimeGetSeconds(item.currentTime()) * 1000 { + currentItemOffset = speechMarks[idx].start ?? 0 break } } @@ -419,17 +473,17 @@ currentItemOffset = (last.start ?? 0) + (last.length ?? 0) } } - + // Sometimes we get negatives currentItemOffset = max(currentItemOffset, 0) - + let idx = currentAudioIndex // item.speechItem.audioIdx if idx < document?.utterances.count ?? 0 { let currentItem = document?.utterances[idx].text ?? "" let currentReadIndex = currentItem.index(currentItem.startIndex, offsetBy: min(currentItemOffset, currentItem.count)) let lastItem = String(currentItem[.. String { UserDefaults.standard.string(forKey: "\(language)-\(UserDefaultKey.textToSpeechPreferredVoice.rawValue)") ?? currentVoiceLanguage.defaultVoice } - + public func setPreferredVoice(_ voice: String, forLanguage language: String) { UserDefaults.standard.set(voice, forKey: "\(language)-\(UserDefaultKey.textToSpeechPreferredVoice.rawValue)") } - + private func downloadAndPlayFrom(_ currentIdx: Int, _ currentOffset: Double) { let desiredState = state - + pause() document = nil synthesizer = nil - + if let itemID = itemAudioProperties?.itemID { Task { let document = try? await self.getSpeechFile(itemID: itemID, priority: .high) - + DispatchQueue.main.async { if let document = document { let synthesizer = SpeechSynthesizer(appEnvironment: self.dataService.appEnvironment, networker: self.dataService.networker, document: document, speechAuthHeader: self.speechAuthHeader) - + self.setTextItems() self.durations = synthesizer.estimatedDurations(forSpeed: self.playbackRate) self.synthesizer = synthesizer - + self.state = desiredState self.synthesizeFrom(start: currentIdx, playWhenReady: self.state == .playing, atOffset: currentOffset) } else { @@ -475,7 +529,7 @@ } } } - + public var secondaryVoice: String { if let pair = currentVoicePair { if pair.firstKey == currentVoice { @@ -487,14 +541,14 @@ } return "en-US-CoraNeural" } - + func previewVoiceURL(_ voice: String) -> URL? { URL(string: "https://storage.googleapis.com/omnivore_preview_bucket/tts-voice-previews/\(voice).mp3") } - + public func playVoiceSample(voice: String) { pause() - + if let url = previewVoiceURL(voice) { samplePlayer = AVPlayer(playerItem: AVPlayerItem(url: url)) if let samplePlayer = samplePlayer { @@ -504,7 +558,7 @@ NSNotification.operationFailed(message: "Error playing voice sample.") } } - + public func isPlayingSample(voice: String) -> Bool { if let samplePlayer = self.samplePlayer, let url = previewVoiceURL(voice) { if let urlAsset = samplePlayer.currentItem?.asset as? AVURLAsset { @@ -514,31 +568,31 @@ } return false } - + public func stopVoiceSample() { if let samplePlayer = self.samplePlayer { samplePlayer.pause() self.samplePlayer = nil } } - + private func updateDurations(oldPlayback: Double, newPlayback: Double) { if let oldDurations = durations { durations = oldDurations.map { $0 * oldPlayback / newPlayback } } } - + public var isLoading: Bool { if state == .reachedEnd { return false } return (state == .loading || player?.currentItem == nil || player?.currentItem?.status == .unknown) } - + public var isPlaying: Bool { state == .playing } - + public func isLoadingItem(_ audioItem: AudioItemProperties?) -> Bool { if state == .reachedEnd { return false @@ -548,41 +602,41 @@ } return itemAudioProperties?.itemID == audioItem?.itemID && isLoading } - + public func isPlayingItem(itemID: String) -> Bool { itemAudioProperties?.itemID == itemID && isPlaying } - + public func skipForward(seconds: Double) { seek(to: timeElapsed + seconds) } - + public func skipBackwards(seconds: Double) { seek(to: timeElapsed - seconds) } - + public func fileNameForAudioFile(_ itemID: String) -> String { itemID + "-" + currentVoice + ".mp3" } - + public static func pathForAudioDirectory(itemID: String) -> URL { URL.om_documentsDirectory .appendingPathComponent("audio-\(itemID)/") } - + public func pathForSpeechFile(itemID: String) -> URL { Self.pathForAudioDirectory(itemID: itemID) .appendingPathComponent("speech-\(currentVoice).json") } - + public func startAudio(atIndex index: Int, andOffset offset: Double) { state = .loading setupNotifications() - + if let itemID = itemAudioProperties?.itemID { Task { let document = try? await getSpeechFile(itemID: itemID, priority: .high) - + DispatchQueue.main.async { self.setTextItems() if let document = document { @@ -598,7 +652,7 @@ } } } - + // swiftlint:disable all private func startStreamingAudio(itemID _: String, document: SpeechDocument, atIndex index: Int, andOffset offset: Double) { do { @@ -608,7 +662,7 @@ // try? FileManager.default.removeItem(atPath: audioUrl.path) state = .stopped } - + player = AVQueuePlayer(items: []) if let player = player { observer = player.observe(\.currentItem, options: [.new]) { _, _ in @@ -616,19 +670,21 @@ self.updateReadText() } } - + let synthesizer = SpeechSynthesizer(appEnvironment: dataService.appEnvironment, networker: dataService.networker, document: document, speechAuthHeader: speechAuthHeader) durations = synthesizer.estimatedDurations(forSpeed: playbackRate) self.synthesizer = synthesizer - + +#if !targetEnvironment(simulator) synthesizeFrom(start: index, playWhenReady: true, atOffset: offset) +#endif } - + func synthesizeFrom(start: Int, playWhenReady: Bool, atOffset: Double = 0.0) { if let synthesizer = self.synthesizer, let items = self.synthesizer?.createPlayerItems(from: start) { let prefetchQueue = OperationQueue() prefetchQueue.maxConcurrentOperationCount = 5 - + for speechItem in items { let isLast = speechItem.audioIdx == synthesizer.document.utterances.count - 1 let playerItem = SpeechPlayerItem(session: self, prefetchQueue: prefetchQueue, speechItem: speechItem) { @@ -655,7 +711,7 @@ } } } - + public func pause() { if let player = player { player.pause() @@ -663,7 +719,7 @@ savePositionInfo(force: true) } } - + public func unpause() { stopVoiceSample() if let player = player { @@ -671,7 +727,7 @@ state = .playing } } - + func formatTimeInterval(_ time: TimeInterval) -> String? { let componentFormatter = DateComponentsFormatter() componentFormatter.unitsStyle = .positional @@ -679,14 +735,14 @@ componentFormatter.zeroFormattingBehavior = .pad return componentFormatter.string(from: time) } - + // What we need is an array of all items in a document, either Utterances if unloaded or AVPlayerItems // if they have been loaded, then for each one we can calculate a duration func durationBefore(playerIndex: Int) -> TimeInterval { let result = durations?.prefix(playerIndex).reduce(0, +) ?? 0 return result } - + func startTimer() { if timer == nil { lastReadUpdate = 0 @@ -694,22 +750,22 @@ timer?.fire() } } - + // Every second, get the current playing time of the player and refresh the status of the player progressslider @objc func fireTimer() { if let player = player { if player.error != nil || player.currentItem?.error != nil { stopWithError() } - + if let durations = durations { duration = durations.reduce(0, +) durationString = formatTimeInterval(duration) } - + updateReadText() } - + if let player = player { switch scrubState { case .reset: @@ -722,10 +778,10 @@ currentAudioIndex = playerItem.speechItem.audioIdx + 1 } } - + timeElapsed = durationBefore(playerIndex: playerItem.speechItem.audioIdx) + itemElapsed timeElapsedString = formatTimeInterval(timeElapsed) - + if var nowPlaying = MPNowPlayingInfoCenter.default().nowPlayingInfo { nowPlaying[MPMediaItemPropertyPlaybackDuration] = NSNumber(value: duration) nowPlaying[MPNowPlayingInfoPropertyElapsedPlaybackTime] = NSNumber(value: timeElapsed) @@ -744,38 +800,38 @@ } } } - + savePositionInfo() } - + func savePositionInfo(force: Bool = false) { if force || (timeElapsed - 10 > lastReadUpdate) { let percentProgress = timeElapsed / duration let speechIndex = (player?.currentItem as? SpeechPlayerItem)?.speechItem.audioIdx ?? 0 let anchorIndex = Int((player?.currentItem as? SpeechPlayerItem)?.speechItem.htmlIdx ?? "") ?? 0 - + if let itemID = itemAudioProperties?.itemID { dataService.updateLinkReadingProgress(itemID: itemID, readingProgress: percentProgress, anchorIndex: anchorIndex, force: true) } - + if let itemID = itemAudioProperties?.itemID, let player = player, let currentItem = player.currentItem { let currentOffset = CMTimeGetSeconds(currentItem.currentTime()) print("updating listening info: ", speechIndex, currentOffset, timeElapsed) - + dataService.updateLinkListeningProgress(itemID: itemID, listenIndex: speechIndex, listenOffset: currentOffset, listenTime: timeElapsed) } - + lastReadUpdate = timeElapsed } } - + func clearNowPlayingInfo() { MPNowPlayingInfoCenter.default().nowPlayingInfo = [:] } - + func downloadAndSetArtwork() async { if let pageId = itemAudioProperties?.itemID, let imageURL = itemAudioProperties?.imageURL { if let result = try? await URLSession.shared.data(from: imageURL) { @@ -795,10 +851,10 @@ } } } - + func setupRemoteControl() { UIApplication.shared.beginReceivingRemoteControlEvents() - + if let itemAudioProperties = itemAudioProperties { MPNowPlayingInfoCenter.default().nowPlayingInfo = [ MPMediaItemPropertyTitle: NSString(string: itemAudioProperties.title), @@ -807,21 +863,21 @@ MPNowPlayingInfoPropertyElapsedPlaybackTime: NSNumber(value: timeElapsed) ] } - + let commandCenter = MPRemoteCommandCenter.shared() - + commandCenter.playCommand.isEnabled = true commandCenter.playCommand.addTarget { _ -> MPRemoteCommandHandlerStatus in self.unpause() return .success } - + commandCenter.pauseCommand.isEnabled = true commandCenter.pauseCommand.addTarget { _ -> MPRemoteCommandHandlerStatus in self.pause() return .success } - + commandCenter.skipForwardCommand.isEnabled = true commandCenter.skipForwardCommand.preferredIntervals = [15, 30, 60] commandCenter.skipForwardCommand.addTarget { event -> MPRemoteCommandHandlerStatus in @@ -831,7 +887,7 @@ } return .commandFailed } - + commandCenter.skipBackwardCommand.isEnabled = true commandCenter.skipBackwardCommand.preferredIntervals = [15, 30, 60] commandCenter.skipBackwardCommand.addTarget { event -> MPRemoteCommandHandlerStatus in @@ -841,7 +897,7 @@ } return .commandFailed } - + commandCenter.changePlaybackPositionCommand.isEnabled = true commandCenter.changePlaybackPositionCommand.addTarget { event -> MPRemoteCommandHandlerStatus in if let event = event as? MPChangePlaybackPositionCommandEvent { @@ -850,12 +906,12 @@ } return .commandFailed } - + Task { await downloadAndSetArtwork() } } - + func isoLangForCurrentVoice() -> String { // currentVoicePair should not ever be nil but if it is we return an empty string if let isoLang = currentVoicePair?.language { @@ -874,11 +930,11 @@ return nil } } - + func downloadLibraryItemSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? { let decoder = JSONDecoder() let speechFileUrl = pathForSpeechFile(itemID: itemID) - + if FileManager.default.fileExists(atPath: speechFileUrl.path) { let data = try Data(contentsOf: speechFileUrl) document = try decoder.decode(SpeechDocument.self, from: data) @@ -887,30 +943,30 @@ return document } } - + let path = "/api/article/\(itemID)/speech?voice=\(currentVoice)&secondaryVoice=\(secondaryVoice)&priority=\(priority)\(isoLangForCurrentVoice())" guard let url = URL(string: path, relativeTo: dataService.appEnvironment.serverBaseURL) else { throw BasicError.message(messageText: "Invalid audio URL") } - + var request = URLRequest(url: url) request.httpMethod = "GET" for (header, value) in dataService.networker.defaultHeaders { request.setValue(value, forHTTPHeaderField: header) } - + let result: (Data, URLResponse)? = try? await URLSession.shared.data(for: request) guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else { throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.") } - + guard let data = result?.0 else { throw BasicError.message(messageText: "audioFetch failed. no data received.") } - + let str = String(decoding: data, as: UTF8.self) print("result speech file: ", str) - + if let document = try? JSONDecoder().decode(SpeechDocument.self, from: data) { do { try? FileManager.default.createDirectory(at: document.audioDirectory, withIntermediateDirectories: true) @@ -920,76 +976,53 @@ print("error writing file", error) } } - + return nil } - + + func combineSpeechFiles(from digest: DigestResult) -> ([Utterance], Double) { + let allUtterances = digest.speechFiles.flatMap { $0.utterances } + var updatedUtterances: [Utterance] = [] + var currentWordOffset = 0.0 + + for (index, utterance) in allUtterances.enumerated() { + let newUtterance = Utterance( + idx: String(index + 1), + text: utterance.text, + voice: utterance.voice, + wordOffset: currentWordOffset, + wordCount: utterance.wordCount + ) + updatedUtterances.append(newUtterance) + currentWordOffset += utterance.wordCount + } + + return (updatedUtterances, currentWordOffset) + } + func downloadDigestItemSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? { - let decoder = JSONDecoder() - let speechFileUrl = URL.om_documentsDirectory.appendingPathComponent("digest").appendingPathComponent("speech-\(currentVoice).json") - - if FileManager.default.fileExists(atPath: speechFileUrl.path) { - let data = try Data(contentsOf: speechFileUrl) - document = try decoder.decode(SpeechDocument.self, from: data) - // If we can't load it from disk we make the API call - if let document = document { - return document - } + if let digestItem = itemAudioProperties as? DigestAudioItem, let firstFile = digestItem.digest.speechFiles.first { + let (utterances, wordCount) = combineSpeechFiles(from: digestItem.digest) + + let document = SpeechDocument( + pageId: digestItem.itemID, + wordCount: wordCount, + language: firstFile.language, + defaultVoice: firstFile.defaultVoice, + utterances: utterances + ) + try? FileManager.default.createDirectory(at: document.audioDirectory, withIntermediateDirectories: true) + return document } - - let path = "/api/digest/v1/" - guard let url = URL(string: path, relativeTo: dataService.appEnvironment.serverBaseURL) else { - throw BasicError.message(messageText: "Invalid audio URL") - } - - var request = URLRequest(url: url) - request.httpMethod = "GET" - for (header, value) in dataService.networker.defaultHeaders { - request.setValue(value, forHTTPHeaderField: header) - } - - let result: (Data, URLResponse)? = try? await URLSession.shared.data(for: request) - guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else { - throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.") - } - - guard let data = result?.0 else { - throw BasicError.message(messageText: "audioFetch failed. no data received.") - } - - let str = String(decoding: data, as: UTF8.self) - print("result digest file: ", str) - - do { - let digest = try JSONDecoder().decode(DigestResult.self, from: data) - let directory = URL.om_documentsDirectory.appendingPathComponent("digest") - // do { - try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - try data.write(to: speechFileUrl) - return digest.speechFile -// } catch { -// print("error writing file", error) -// } - // } - } catch { - print("error with digest file", error) - } - + return nil } - + func getSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? { document = try await downloadSpeechFile(itemID: itemID, priority: priority) return document } - - public func audioPlayerDidFinishPlaying(_ player: AVAudioPlayer, successfully _: Bool) { - if player == self.player { - pause() - player.currentTime = 0 - } - } - + func setupNotifications() { NotificationCenter.default.removeObserver(self, name: AVAudioSession.interruptionNotification, object: AVAudioSession.sharedInstance()) NotificationCenter.default.addObserver(self, @@ -997,7 +1030,7 @@ name: AVAudioSession.interruptionNotification, object: AVAudioSession.sharedInstance()) } - + @objc func handleInterruption(notification: Notification) { guard let userInfo = notification.userInfo, let typeValue = userInfo[AVAudioSessionInterruptionTypeKey] as? UInt, @@ -1005,7 +1038,7 @@ else { return } - + // Switch over the interruption type. switch type { case .began: @@ -1013,7 +1046,6 @@ pause() case .ended: // An interruption ended. Resume playback, if appropriate. - guard let optionsValue = userInfo[AVAudioSessionInterruptionOptionKey] as? UInt else { return } let options = AVAudioSession.InterruptionOptions(rawValue: optionsValue) if options.contains(.shouldResume) { @@ -1022,6 +1054,6 @@ default: () } } - } +} #endif diff --git a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift index 2946c0fb0..a48342668 100644 --- a/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift +++ b/apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift @@ -20,7 +20,7 @@ struct UtteranceRequest: Codable { let isOpenAIVoice: Bool } -public struct Utterance: Decodable { +public struct Utterance: Codable { public let idx: String public let text: String public let voice: String? @@ -39,10 +39,10 @@ public struct Utterance: Decodable { } } -public struct SpeechDocument: Decodable { - static let averageWPM: Double = 195 +public struct SpeechDocument: Codable { + public static let averageWPM: Double = 195 - public let pageId: String + public let pageId: String? public let wordCount: Double public let language: String public let defaultVoice: String @@ -54,7 +54,7 @@ public struct SpeechDocument: Decodable { } var audioDirectory: URL { - Self.audioDirectory(pageId: pageId) + Self.audioDirectory(pageId: pageId ?? "pageid") } static func audioDirectory(pageId: String) -> URL { diff --git a/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift b/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift index 7ea58022c..130aa538e 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift @@ -7,17 +7,40 @@ struct AITaskRequest: Decodable { public let requestId: String } -public struct DigestResult: Decodable { +public struct DigestResult: Codable { public let id: String public let title: String + public let byline: String public let content: String + public let description: String public let urlsToAudio: [String] - public let speechFile: SpeechDocument + public let chapters: [DigestChapter] + public let speechFiles: [SpeechDocument] public let jobState: String + public let createdAt: String } -public struct DigestItem: Decodable { +public struct DigestChapter: Codable { + public let title: String + public let id: String + public let url: String + public let wordCount: Double + public let thumbnail: String? + public init(title: String, id: String, url: String, wordCount: Double, thumbnail: String?) { + self.title = title + self.id = id + self.url = url + self.wordCount = wordCount + self.thumbnail = thumbnail + } +} + +public struct RefreshDigestResult: Codable { + public let jobId: String +} + +public struct DigestItem: Codable { public let id: String public let site: String public let siteIcon: URL? @@ -26,7 +49,7 @@ public struct DigestItem: Decodable { public let summaryText: String public let keyPointsText: String public let highlightsText: String - public init(id: String, site: String, siteIcon: URL?, + public init(id: String, site: String, siteIcon: URL?, author: String, title: String, summaryText: String, keyPointsText: String, highlightsText: String) { self.id = id @@ -40,32 +63,53 @@ public struct DigestItem: Decodable { } } +public struct DigestRequest: Codable { + public let schedule: String + public let voices: [String] + public init(schedule: String, voices: [String]) { + self.schedule = schedule + self.voices = voices + } +} + +public struct ExplainRequest: Codable { + public let text: String + public let libraryItemId: String + public init(text: String, libraryItemId: String) { + self.text = text + self.libraryItemId = libraryItemId + } +} + +public struct ExplainResult: Codable { + public let text: String +} + extension DataService { -// public func createAITask(extraText: String?, libraryItemId: String, promptName: String) async throws -> String? { -// let jsonData = try JSONSerialization.data(withJSONObject: [ -// "libraryItemId": libraryItemId, -// "promptName": promptName, -// "extraText": extraText -// ]) -// -// let urlRequest = URLRequest.create( -// baseURL: appEnvironment.serverBaseURL, -// urlPath: "/api/ai-task", -// requestMethod: .post(params: jsonData), -// includeAuthToken: true -// ) -// let resource = ServerResource( -// urlRequest: urlRequest, -// decode: AITaskRequest.decode -// ) -// -// do { -// let taskRequest = try await networker.urlSession.performRequest(resource: resource) -// return taskRequest.requestId -// } catch { -// return nil -// } -// } + public func refreshDigest() async throws { + let encoder = JSONEncoder() + let digestRequest = DigestRequest(schedule: "daily", voices: ["openai-nova"]) + let data = (try? encoder.encode(digestRequest)) ?? Data() + + let urlRequest = URLRequest.create( + baseURL: appEnvironment.serverBaseURL, + urlPath: "/api/digest/v1/", + requestMethod: .post(params: data), + includeAuthToken: true + ) + + let resource = ServerResource( + urlRequest: urlRequest, + decode: RefreshDigestResult.decode + ) + + do { + let digest = try await networker.urlSession.performRequest(resource: resource) + print("GOT RESPONSE: ", digest) + } catch { + print("ERROR FETCHING TASK: ", error) + } + } // Function to poll the status of the AI task with timeout public func getLatestDigest(timeoutInterval: TimeInterval) async throws -> DigestResult? { @@ -76,43 +120,81 @@ extension DataService { if count > 3 { return nil } - do { - // Check if timeout has occurred - if -startTime.timeIntervalSinceNow >= timeoutInterval { - throw NSError(domain: "Timeout Error", code: -1, userInfo: nil) - } - - let urlRequest = URLRequest.create( - baseURL: appEnvironment.serverBaseURL, - urlPath: "/api/digest/v1/", - requestMethod: .get, - includeAuthToken: true - ) - - let resource = ServerResource( - urlRequest: urlRequest, - decode: DigestResult.decode - ) - - do { - let digest = try await networker.urlSession.performRequest(resource: resource) - print("GOT RESPONSE: ", digest) - return digest - } catch { - print("ERROR FETCHING TASK: ", error) -// if let response = error as? ServerError { -// if response != .stillProcessing { -// return nil -// } -// } - } - // Wait for some time before polling again - try? await Task.sleep(nanoseconds: 3_000_000_000) - } catch let error { - throw error + do { + // Check if timeout has occurred + if -startTime.timeIntervalSinceNow >= timeoutInterval { + throw NSError(domain: "Timeout Error", code: -1, userInfo: nil) } + + let urlRequest = URLRequest.create( + baseURL: appEnvironment.serverBaseURL, + urlPath: "/api/digest/v1/", + requestMethod: .get, + includeAuthToken: true + ) + + let resource = ServerResource( + urlRequest: urlRequest, + decode: DigestResult.decode + ) + + do { + let digest = try await networker.urlSession.performRequest(resource: resource) + let oldDigest = loadStoredDigest() + + saveDigest(digest) + + return digest + } catch { + print("ERROR FETCHING TASK: ", error) + } + // Wait for some time before polling again + try? await Task.sleep(nanoseconds: 3_000_000_000) + } catch let error { + throw error + } } } + + public func loadStoredDigest() -> DigestResult? { + let decoder = JSONDecoder() + let localPath = URL.om_cachesDirectory.appendingPathComponent("digest.json") + if let data = try? Data(contentsOf: localPath), + let digest = try? decoder.decode(DigestResult.self, from: data) { + return digest + } + return nil + } + + func saveDigest(_ digest: DigestResult) { + let localPath = URL.om_cachesDirectory.appendingPathComponent("digest.json") + if let data = try? JSONEncoder().encode(digest) { + try? data.write(to: localPath) + } + } + + public func explain(text: String, libraryItemId: String) async throws -> String { + let encoder = JSONEncoder() + let explainRequest = ExplainRequest(text: text, libraryItemId: libraryItemId) + let data = (try? encoder.encode(explainRequest)) ?? Data() + + do { + let urlRequest = URLRequest.create( + baseURL: appEnvironment.serverBaseURL, + urlPath: "/api/explain/", + requestMethod: .post(params: data), + includeAuthToken: true + ) + + let resource = ServerResource( + urlRequest: urlRequest, + decode: ExplainResult.decode + ) + + let response = try await networker.urlSession.performRequest(resource: resource) + return response.text + } catch let error { + throw error + } + } } - - diff --git a/apple/OmnivoreKit/Sources/Services/DataService/DataService.swift b/apple/OmnivoreKit/Sources/Services/DataService/DataService.swift index 11062d9ea..0b131f3c7 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/DataService.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/DataService.swift @@ -33,6 +33,8 @@ public final class DataService: ObservableObject { persistentContainer.viewContext } + public var featureFlags = FeatureFlags() + public var lastItemSyncTime: Date { get { guard @@ -298,4 +300,11 @@ public final class DataService: ObservableObject { } return objectID } + + public func tryUpdateFeatureFlags() async { + if let features = (try? await fetchViewer())?.enabledFeatures { + featureFlags.digestEnabled = features.contains("ai-digest") + featureFlags.explainEnabled = features.contains("ai-explain") + } + } } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift index 660d08fda..4c7605e93 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLink.swift @@ -35,7 +35,7 @@ public extension DataService { if let linkedItemID = linkedItemID { Task { - AudioController.removeAudioFiles(itemID: linkedItemID) + await AudioController.removeAudioFiles(itemID: linkedItemID) } } } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift index 2ed12e99f..f5365ebc1 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift @@ -168,6 +168,7 @@ extension DataService { try $0.search( after: OptionalArgument(cursor), first: OptionalArgument(limit), + includeContent: OptionalArgument(true), query: OptionalArgument(searchQuery), selection: selection ) diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift index f1251fdc0..c386b991d 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift @@ -18,8 +18,7 @@ public extension DataService { selection: .init { try $0.pictureUrl() } ), intercomHash: try $0.intercomHash(), - digestEnabled: true // (try $0.featureList(selection: featureSelection.list.nullable)? - // .filter { $0.enabled && $0.name == "digest" } ?? []).count > 0 + enabledFeatures: try $0.featureList(selection: featureSelection.list.nullable)?.filter { $0.enabled }.map { $0.name } ) } @@ -67,7 +66,11 @@ public struct ViewerInternal { public let name: String public let profileImageURL: String? public let intercomHash: String? - public let digestEnabled: Bool? + public let enabledFeatures: [String]? // We don't persist these as they can be dynamic + + public func hasFeatureGranted(_ name: String) -> Bool { + return enabledFeatures?.contains(name) ?? false + } func persist(context: NSManagedObjectContext) throws { try context.performAndWait { @@ -76,7 +79,6 @@ public struct ViewerInternal { viewer.username = username viewer.name = name viewer.profileImageURL = profileImageURL - viewer.digestEnabled = digestEnabled ?? false do { try context.save() diff --git a/apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift b/apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift index 9f0d5f898..24b11cd7d 100644 --- a/apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift +++ b/apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift @@ -1,4 +1,5 @@ import Foundation +import SwiftUI #if DEBUG public let isDebug = true @@ -6,11 +7,13 @@ import Foundation public let isDebug = false #endif -public enum FeatureFlag { - public static let enableSnoozeFromShareExtension = false - public static let enableRemindersFromShareExtension = false - public static let enableShareButton = false - public static let enableSnooze = false - public static let enableGridCardsOnPhone = false - public static let enableUltraRealisticVoices = true +public struct FeatureFlags { + @AppStorage("FeatureFlag::digestEnabled") + public var digestEnabled = false + + @AppStorage("FeatureFlag::explainEnabled") + public var explainEnabled = false + + public init() { + } } diff --git a/apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift b/apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift index ee3758a02..04bfbd1e5 100644 --- a/apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift +++ b/apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift @@ -38,4 +38,5 @@ public enum UserDefaultKey: String { case openExternalLinksIn case prefersHideStatusBarInReader case visibleShareExtensionTab + case lastVisitedDigestId } diff --git a/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift b/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift index 05dcfb44d..93ddf7910 100644 --- a/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift +++ b/apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift @@ -27,6 +27,8 @@ public final class OmnivoreWebView: WKWebView { private var currentMenu: ContextMenu = .defaultMenu + private var explainEnabled = false + override init(frame: CGRect, configuration: WKWebViewConfiguration) { super.init(frame: frame, configuration: configuration) @@ -340,7 +342,6 @@ public final class OmnivoreWebView: WKWebView { Task { let selection = try? await self.evaluateJavaScript("window.getSelection().toString()") if let selection = selection as? String, let explainHandler = explainHandler { - print("Explaining \(selection)") explainHandler(selection) } else { showInReaderSnackbar("Error getting text to explain") @@ -400,8 +401,12 @@ public final class OmnivoreWebView: WKWebView { return } let highlight = UICommand(title: LocalText.genericHighlight, action: #selector(highlightSelection)) - // let explain = UICommand(title: "Explain", action: #selector(explainSelection)) - items = [highlight, /* explain, */ annotate] + if explainHandler != nil { + let explain = UICommand(title: "Explain", action: #selector(explainSelection)) + items = [highlight, explain, annotate] + } else { + items = [highlight, annotate] + } } else { let remove = UICommand(title: "Remove", action: #selector(removeSelection)) let setLabels = UICommand(title: LocalText.labelsGeneric, action: #selector(setLabels)) diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.swift b/apple/OmnivoreKit/Sources/Views/Images/Images.swift index 9b684783e..ccfb85f88 100644 --- a/apple/OmnivoreKit/Sources/Views/Images/Images.swift +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.swift @@ -2,6 +2,7 @@ import SwiftUI public extension Image { static var smallOmnivoreLogo: Image { Image("_smallOmnivoreLogo", bundle: .module) } + static var coloredSmallOmnivoreLogo: Image { Image("app-icon", bundle: .module) } static var omnivoreTitleLogo: Image { Image("_omnivoreTitleLogo", bundle: .module) } static var googleIcon: Image { Image("_googleIcon", bundle: .module) } diff --git a/packages/api/src/routers/explain_router.ts b/packages/api/src/routers/explain_router.ts new file mode 100644 index 000000000..347807889 --- /dev/null +++ b/packages/api/src/routers/explain_router.ts @@ -0,0 +1,58 @@ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler' +import cors from 'cors' +import express from 'express' +import { userRepository } from '../repository/user' +import { getClaimsByToken } from '../utils/auth' +import { corsConfig } from '../utils/corsConfig' +import { getAISummary } from '../services/ai-summaries' +import { explainText } from '../services/explain' +import { FeatureName, findGrantedFeatureByName } from '../services/features' + +export function explainRouter() { + const router = express.Router() + + // Get an indexed summary for an individual library item + router.post('/', cors(corsConfig), async (req, res) => { + const token = req?.cookies?.auth || req?.headers?.authorization + const claims = await getClaimsByToken(token) + if (!claims) { + return res.status(401).send('UNAUTHORIZED') + } + + const { uid } = claims + const user = await userRepository.findById(uid) + if (!user) { + return res.status(400).send('Bad Request') + } + + if (!(await findGrantedFeatureByName(FeatureName.AIExplain, user.id))) { + return res.status(403).send('Not granted') + } + + const libraryItemId = req.body.libraryItemId + if (!libraryItemId) { + return res.status(400).send('Bad request - no library item id provided') + } + + const text = req.body.text + if (!text) { + return res.status(400).send('Bad request - no idx provided') + } + + try { + const result = await explainText(uid, text, libraryItemId) + + return res.send({ + text: result, + }) + } catch (err) { + console.log('Error: ', err) + } + + return res.status(500).send('Error') + }) + + return router +} diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 87d12536a..3fa1fdb9e 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -44,6 +44,7 @@ import { analytics } from './utils/analytics' import { corsConfig } from './utils/corsConfig' import { buildLogger, buildLoggerTransport, logger } from './utils/logger' import { apiLimiter, authLimiter } from './utils/rate_limit' +import { explainRouter } from './routers/explain_router' const PORT = process.env.PORT || 4000 @@ -85,6 +86,7 @@ export const createApp = (): Express => { app.use('/api/user', userRouter()) app.use('/api/article', articleRouter()) app.use('/api/ai-summary', aiSummariesRouter()) + app.use('/api/explain', explainRouter()) app.use('/api/text-to-speech', textToSpeechRouter()) app.use('/api/notification', notificationRouter()) app.use('/api/integration', integrationRouter()) diff --git a/packages/api/src/services/create_user.ts b/packages/api/src/services/create_user.ts index c71da06cd..a52b76c22 100644 --- a/packages/api/src/services/create_user.ts +++ b/packages/api/src/services/create_user.ts @@ -37,7 +37,7 @@ export const createUser = async (input: { const existingUser = await userRepository.findByEmail(trimmedEmail) if (existingUser) { if (existingUser.profile) { - return Promise.reject({ errorCode: SignupErrorCode.UserExists }) + return Promise.reject({ errorCode: SignupErrorCode.Unknown }) } // create profile if user exists but profile does not exist diff --git a/packages/api/src/services/explain.ts b/packages/api/src/services/explain.ts new file mode 100644 index 000000000..407f2b230 --- /dev/null +++ b/packages/api/src/services/explain.ts @@ -0,0 +1,52 @@ +import { OpenAI } from '@langchain/openai' +import { PromptTemplate } from '@langchain/core/prompts' +import { authTrx } from '../repository' +import { libraryItemRepository } from '../repository/library_item' +import { htmlToMarkdown } from '../utils/parser' + +export const explainText = async ( + userId: string, + text: string, + libraryItemId: string +): Promise => { + const llm = new OpenAI({ + modelName: 'gpt-4-0125-preview', + configuration: { + apiKey: process.env.OPENAI_API_KEY, + }, + }) + + const libraryItem = await authTrx( + async (tx) => + tx.withRepository(libraryItemRepository).findById(libraryItemId), + undefined, + userId + ) + + if (!libraryItem) { + throw 'No library item found' + } + + const content = htmlToMarkdown(libraryItem.readableContent) + + const contextualTemplate = PromptTemplate.fromTemplate( + `Create a brief, less than 300 character explanation of the provided + term. Use the article text for additional context. + + Term: {text} + + Article text: {content} + ` + ) + + console.log('template: ', contextualTemplate) + + const chain = contextualTemplate.pipe(llm) + const result = await chain.invoke({ + text: text, + content, + }) + console.log('result: ', result) + + return result +} diff --git a/packages/api/src/services/features.ts b/packages/api/src/services/features.ts index 6400c4779..709615e18 100644 --- a/packages/api/src/services/features.ts +++ b/packages/api/src/services/features.ts @@ -17,6 +17,7 @@ export enum FeatureName { UltraRealisticVoice = 'ultra-realistic-voice', Notion = 'notion', AIDigest = 'ai-digest', + AIExplain = 'ai-explain', } export const getFeatureName = (name: string): FeatureName | undefined => { diff --git a/packages/api/test/routers/auth.test.ts b/packages/api/test/routers/auth.test.ts index 8bfd3e5e4..32b438047 100644 --- a/packages/api/test/routers/auth.test.ts +++ b/packages/api/test/routers/auth.test.ts @@ -90,12 +90,12 @@ describe('auth router', () => { await deleteUser(user.id) }) - it('redirects to sign up page with error code USER_EXISTS', async () => { + it('redirects to sign up page with error code UNKNOWN', async () => { const res = await signupRequest(email, password, name, username).expect( 302 ) expect(res.header.location).to.endWith( - '/email-signup?errorCodes=USER_EXISTS' + '/email-signup?errorCodes=UNKNOWN' ) }) }) diff --git a/packages/web/components/elements/icons/MarkAsReadIcon.tsx b/packages/web/components/elements/icons/MarkAsReadIcon.tsx new file mode 100644 index 000000000..bb951b35d --- /dev/null +++ b/packages/web/components/elements/icons/MarkAsReadIcon.tsx @@ -0,0 +1,39 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class MarkAsReadIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + // tick letters button + return ( + + + + + + + ) + } +} diff --git a/packages/web/components/templates/homeFeed/HomeFeedContainer.tsx b/packages/web/components/templates/homeFeed/HomeFeedContainer.tsx index 2eb4bef6b..cc6014c41 100644 --- a/packages/web/components/templates/homeFeed/HomeFeedContainer.tsx +++ b/packages/web/components/templates/homeFeed/HomeFeedContainer.tsx @@ -761,6 +761,7 @@ export function HomeFeedContainer(): JSX.Element { ) if (res) { let successMessage: string | undefined = undefined + console.log(action) switch (action) { case BulkAction.ARCHIVE: successMessage = 'Link Archived' @@ -771,6 +772,9 @@ export function HomeFeedContainer(): JSX.Element { case BulkAction.DELETE: successMessage = 'Items deleted' break + case BulkAction.MARK_AS_READ: + successMessage = 'Items marked as read' + break } if (successMessage) { showSuccessToast(successMessage, { position: 'bottom-right' }) diff --git a/packages/web/components/templates/homeFeed/MultiSelectControls.tsx b/packages/web/components/templates/homeFeed/MultiSelectControls.tsx index 987889182..e1cbb2113 100644 --- a/packages/web/components/templates/homeFeed/MultiSelectControls.tsx +++ b/packages/web/components/templates/homeFeed/MultiSelectControls.tsx @@ -12,6 +12,7 @@ import { X } from 'phosphor-react' import { LibraryHeaderProps } from './LibraryHeader' import { HeaderCheckboxIcon } from '../../elements/icons/HeaderCheckboxIcon' import { Label } from '../../../lib/networking/fragments/labelFragment' +import { MarkAsReadIcon } from '../../elements/icons/MarkAsReadIcon' export const MultiSelectControls = (props: LibraryHeaderProps): JSX.Element => { const [showConfirmDelete, setShowConfirmDelete] = useState(false) @@ -114,6 +115,7 @@ export const MultiSelectControls = (props: LibraryHeaderProps): JSX.Element => { + {showConfirmDelete && ( { ) } +export const MarkAsReadButton = (props: LibraryHeaderProps): JSX.Element => { + const [color, setColor] = useState( + theme.colors.thTextContrast2.toString() + ) + return ( + + ) +} + type AddLabelsButtonProps = { setShowLabelsModal: (set: boolean) => void } diff --git a/packages/web/lib/networking/mutations/bulkActionMutation.ts b/packages/web/lib/networking/mutations/bulkActionMutation.ts index 71d02a281..fd02e66c0 100644 --- a/packages/web/lib/networking/mutations/bulkActionMutation.ts +++ b/packages/web/lib/networking/mutations/bulkActionMutation.ts @@ -5,6 +5,7 @@ export enum BulkAction { ARCHIVE = 'ARCHIVE', DELETE = 'DELETE', ADD_LABELS = 'ADD_LABELS', + MARK_AS_READ = 'MARK_AS_READ', } type BulkActionResponseData = {