prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/main' into fix/android-library-flickering

Browse source
This commit is contained in:
Stefano Sansone 2024-04-03 10:05:34 +02:00
commit 89dfc98622
326 changed files with 21958 additions and 3239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
.github/workflows/run-tests.yaml vendored
View file

@ -104,17 +104,8 @@ jobs:
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: 'Login to GitHub container registry'
uses: docker/login-action@v1
with:
registry: ghcr.io
username: ${{github.actor}}
password: ${{secrets.GITHUB_TOKEN}}
- name: Build the API docker image
# run: 'docker build --file packages/api/Dockerfile .'
run: |
docker build . --file packages/api/Dockerfile --tag "ghcr.io/omnivore-app/backend:${GITHUB_SHA}" --tag ghcr.io/omnivore-app/backend:latest
docker push ghcr.io/omnivore-app/backend:${GITHUB_SHA}
run: 'docker build --file packages/api/Dockerfile .'
- name: Build the content-fetch docker image
run: 'docker build --file packages/content-fetch/Dockerfile .'
- name: Build the inbound-email-handler docker image

2
README.md
View file

@ -1,6 +1,6 @@
# Omnivore
[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/omnivore-app/omnivore/Run%20tests)](https://github.com/omnivore-app/omnivore/actions/workflows/run-tests.yaml)
[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/omnivore-app/omnivore/run-tests.yaml?branch=main)](https://github.com/omnivore-app/omnivore/actions/workflows/run-tests.yaml)
[![Discord](https://img.shields.io/discord/844965259462311966?label=Join%20our%20Discord)](https://discord.gg/h2z5rppzz9)
[![Mastodon Follow](https://img.shields.io/mastodon/follow/109458738600914558?domain=https%3A%2F%2Fpkm.social)](https://pkm.social/@omnivore)
[![Twitter Follow](https://img.shields.io/twitter/follow/omnivoreapp)](https://twitter.com/OmnivoreApp)

2
android/Omnivore/app/build.gradle.kts
View file

@ -28,7 +28,7 @@ android {
minSdk = 26
targetSdk = 34
versionCode = 194001
versionName = "0.194.1"
versionName = "0.195.0"
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
vectorDrawables {

1
android/Omnivore/app/src/main/AndroidManifest.xml
View file

@ -4,7 +4,6 @@
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/>
<uses-permission android:name="com.google.android.gms.permission.AD_ID"/>
<uses-permission android:name="android.permission.WAKE_LOCK" />
<application

2
android/Omnivore/app/src/main/assets/bundle.js
View file

File diff suppressed because one or more lines are too long

17
android/Omnivore/app/src/main/java/app/omnivore/omnivore/feature/reader/WebReaderLoadingContainer.kt
View file

@ -6,6 +6,7 @@ import android.view.View
import androidx.activity.ComponentActivity
import androidx.activity.compose.LocalOnBackPressedDispatcherOwner
import androidx.activity.compose.setContent
import androidx.activity.enableEdgeToEdge
import androidx.activity.viewModels
import androidx.compose.foundation.background
import androidx.compose.foundation.isSystemInDarkTheme
@ -78,12 +79,14 @@ class WebReaderLoadingContainerActivity : ComponentActivity() {
viewModel.loadItem(slug = slug, requestID = requestID)
enableEdgeToEdge()
setContent {
OmnivoreTheme {
Box(
modifier = Modifier
.fillMaxSize()
.background(color = Color.Black)
.fillMaxSize()
.background(color = if (isSystemInDarkTheme()) Color.Black else Color.White)
) {
if (viewModel.hasFetchError.value == true) {
Text(stringResource(R.string.web_reader_loading_container_error_msg))
@ -499,11 +502,11 @@ fun ReaderTopAppBar(
fun BottomSheetUI(content: @Composable () -> Unit) {
Box(
modifier = Modifier
.wrapContentHeight()
.fillMaxWidth()
.clip(RoundedCornerShape(topEnd = 20.dp, topStart = 20.dp))
.background(Color.White)
.statusBarsPadding()
.wrapContentHeight()
.fillMaxWidth()
.clip(RoundedCornerShape(topEnd = 20.dp, topStart = 20.dp))
.background(Color.White)
.statusBarsPadding()
) {
Scaffold { paddingValues ->
Box(modifier = Modifier.fillMaxSize()) {

8
apple/Omnivore.xcodeproj/project.pbxproj
View file

@ -1389,7 +1389,7 @@
"@executable_path/../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 12.0;
MARKETING_VERSION = 1.45.0;
MARKETING_VERSION = 1.46.0;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
@ -1424,7 +1424,7 @@
"@executable_path/../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 12.0;
MARKETING_VERSION = 1.45.0;
MARKETING_VERSION = 1.46.0;
MTL_FAST_MATH = YES;
PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
PRODUCT_NAME = "$(TARGET_NAME)";
@ -1479,7 +1479,7 @@
"$(inherited)",
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.45.0;
MARKETING_VERSION = 1.46.0;
PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
PRODUCT_NAME = Omnivore;
PROVISIONING_PROFILE_SPECIFIER = "";
@ -1820,7 +1820,7 @@
"$(inherited)",
"@executable_path/Frameworks",
);
MARKETING_VERSION = 1.45.0;
MARKETING_VERSION = 1.46.0;
PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app;
PRODUCT_NAME = Omnivore;
PROVISIONING_PROFILE_SPECIFIER = "";

26
apple/Omnivore.xcworkspace/xcshareddata/swiftpm/Package.resolved
View file

@ -32,8 +32,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/nathantannar4/Engine",
"state" : {
"revision" : "31949c114698e4fd43fd76290913bca415fa87bc",
"version" : "1.1.0"
"revision" : "e9867eb6df013abc65c3437d295e594077469a13",
"version" : "1.5.1"
}
},
{
@ -180,24 +180,6 @@
"version" : "1.0.2"
}
},
{
"identity" : "swift-async-algorithms",
"kind" : "remoteSourceControl",
"location" : "https://github.com/apple/swift-async-algorithms",
"state" : {
"revision" : "da4e36f86544cdf733a40d59b3a2267e3a7bbf36",
"version" : "1.0.0"
}
},
{
"identity" : "swift-collections",
"kind" : "remoteSourceControl",
"location" : "https://github.com/apple/swift-collections.git",
"state" : {
"revision" : "d029d9d39c87bed85b1c50adee7c41795261a192",
"version" : "1.0.6"
}
},
{
"identity" : "swift-graphql",
"kind" : "remoteSourceControl",
@ -266,8 +248,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/nathantannar4/Turbocharger",
"state" : {
"revision" : "b4201ba0bc094facf6cabe3b36fd3763b51ccfc8",
"version" : "1.0.1"
"revision" : "095344c0cac57873e1552f30d3561ab1bec5ae35",
"version" : "1.1.4"
}
},
{

5
apple/OmnivoreKit/Package.swift
View file

@ -71,8 +71,9 @@ var dependencies: [Package.Dependency] {
.package(url: "https://github.com/google/GoogleSignIn-iOS", from: "6.2.2"),
.package(url: "https://github.com/gonzalezreal/swift-markdown-ui", from: "2.0.0"),
.package(url: "https://github.com/PostHog/posthog-ios.git", from: "2.0.0"),
.package(url: "https://github.com/nathantannar4/Transmission", from: "1.0.1"),
.package(url: "https://github.com/apple/swift-async-algorithms", from: "1.0.0")
// .package(url: "https://github.com/nathantannar4/Engine", exact: "1.0.1"),
// .package(url: "https://github.com/nathantannar4/Turbocharger", exact: "1.1.4"),
.package(url: "https://github.com/nathantannar4/Transmission", exact: "1.0.1")
]
// Comment out following line for macOS build
deps.append(.package(url: "https://github.com/PSPDFKit/PSPDFKit-SP", from: "13.1.0"))

6
apple/OmnivoreKit/Sources/App/AppExtensions/Share/ShareExtensionViewModel.swift
View file

@ -38,6 +38,12 @@ public class ShareExtensionViewModel: ObservableObject {
extensionContext?.completeRequest(returningItems: [], completionHandler: nil)
}
public func dismissExtension(extensionContext: NSExtensionContext?) {
if let extensionContext = extensionContext {
extensionContext.completeRequest(returningItems: [], completionHandler: nil)
}
}
func savePage(extensionContext: NSExtensionContext?) {
if let extensionContext = extensionContext {
save(extensionContext)

2
apple/OmnivoreKit/Sources/App/AppExtensions/Share/Views/ShareExtensionView.swift
View file

@ -323,7 +323,7 @@ public struct ShareExtensionView: View {
#endif
Spacer()
Button(action: {
extensionContext?.completeRequest(returningItems: [], completionHandler: nil)
viewModel.dismissExtension(extensionContext: extensionContext)
}, label: {
Text("Dismiss")
#if os(iOS)

1
apple/OmnivoreKit/Sources/App/Views/AudioPlayer/AudioToolbarButton.swift
View file

@ -1,4 +1,3 @@
import Foundation
import SwiftUI

5
apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ExpandedAudioPlayer.swift
View file

@ -385,7 +385,10 @@
func playerContent(_: LinkedItemAudioProperties) -> some View {
ZStack {
WindowLink(level: .alert, transition: .move(edge: .bottom), isPresented: $showSnackbar) {
OperationToast(operationMessage: $snackbarMessage, showOperationToast: $showSnackbar, operationStatus: $operationStatus)
OperationToast(
operationMessage: $snackbarMessage,
showOperationToast: $showSnackbar,
operationStatus: $operationStatus)
.offset(y: -90)
} label: {
EmptyView()

1
apple/OmnivoreKit/Sources/App/Views/Home/Components/LibraryItemFetcher.swift
View file

@ -1,4 +1,3 @@
import Foundation
import CoreData

6
apple/OmnivoreKit/Sources/App/Views/Home/Components/LibraryItemListNavigationLink.swift
View file

@ -14,7 +14,7 @@ struct MacFeedCardNavigationLink: View {
var body: some View {
ZStack {
LibraryItemCard(item: LibraryItemData.make(from: item), viewer: dataService.currentViewer)
LibraryItemCard(item: item, viewer: dataService.currentViewer)
NavigationLink(destination: LinkItemDetailView(
linkedItemObjectID: item.objectID,
isPDF: item.isPDF
@ -36,7 +36,7 @@ struct LibraryItemListNavigationLink: View {
Button(action: {
viewModel.presentItem(item: item)
}, label: {
LibraryItemCard(item: LibraryItemData.make(from: item), viewer: dataService.currentViewer)
LibraryItemCard(item: item, viewer: dataService.currentViewer)
})
}
}
@ -54,7 +54,7 @@ struct LibraryItemGridCardNavigationLink: View {
Button(action: {
viewModel.presentItem(item: item)
}, label: {
GridCard(item: LibraryItemData.make(from: item))
GridCard(item: item)
})
.buttonStyle(.plain)
.aspectRatio(1.0, contentMode: .fill)

121
apple/OmnivoreKit/Sources/App/Views/Home/FilterSelectorView.swift
View file

@ -1,121 +0,0 @@
// import Introspect
// import Models
// import Services
// import SwiftUI
// import Views
//
// @MainActor final class FilterSelectorViewModel: NSObject, ObservableObject {
// @Published var isLoading = false
// @Published var errorMessage: String = ""
// @Published var showErrorMessage: Bool = false
//
// func error(_ msg: String) {
// errorMessage = msg
// showErrorMessage = true
// isLoading = false
// }
// }
//
// struct FilterSelectorView: View {
// @ObservedObject var viewModel: HomeFeedViewModel
// @ObservedObject var filterViewModel = FilterByLabelsViewModel()
// @EnvironmentObject var dataService: DataService
// @Environment(\.dismiss) private var dismiss
//
// @State var showLabelsSheet = false
//
// init(viewModel: HomeFeedViewModel) {
// self.viewModel = viewModel
// }
//
// var body: some View {
// Group {
// #if os(iOS)
// List {
// innerBody
// }
// .listStyle(.grouped)
// #elseif os(macOS)
// List {
// innerBody
// }
// .listStyle(.plain)
// #endif
// }
// #if os(iOS)
// .navigationBarTitle("Library")
// .navigationBarTitleDisplayMode(.inline)
// .navigationBarItems(trailing: doneButton)
// #endif
// }
//
// private var innerBody: some View {
// Group {
// Section {
// ForEach(LinkedItemFilter.allCases, id: \.self) { filter in
// HStack {
// Text(filter.displayName)
// .foregroundColor(filterState.appliedFilter == filter.rawValue ? Color.blue : Color.appTextDefault)
// Spacer()
// if filterState.appliedFilter == filter.rawValue {
// Image(systemName: "checkmark")
// .foregroundColor(Color.blue)
// }
// }
// .contentShape(Rectangle())
// .onTapGesture {
// filterState.appliedFilter = filter.rawValue
// }
// }
// }
//
// Section("Labels") {
// Button(
// action: {
// showLabelsSheet = true
// },
// label: {
// HStack {
// Text("Select Labels (\(filterState.selectedLabels.count))")
// Spacer()
// Image(systemName: "chevron.right")
// }
// }
// )
// }
// }
// .sheet(isPresented: $showLabelsSheet) {
// FilterByLabelsView(
// initiallySelected: filterState.selectedLabels,
// initiallyNegated: filterState.negatedLabels
// ) {
// self.filterState.selectedLabels = $0
// self.filterState.negatedLabels = $1
// }
// }
// .task {
// await filterViewModel.loadLabels(
// dataService: dataService,
// initiallySelectedLabels: filterState.selectedLabels,
// initiallyNegatedLabels: filterState.negatedLabels
// )
// }
// }
//
// func isNegated(_ label: LinkedItemLabel) -> Bool {
// filterViewModel.negatedLabels.contains(where: { $0.id == label.id })
// }
//
// func isSelected(_ label: LinkedItemLabel) -> Bool {
// filterViewModel.selectedLabels.contains(where: { $0.id == label.id })
// }
//
// var doneButton: some View {
// Button(
// action: { dismiss() },
// label: { Text("Done") }
// )
// .disabled(viewModel.isLoading)
// }
// }

78
apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift
View file

@ -150,13 +150,15 @@ struct EmptyState: View {
return AnyView(Group {
Spacer()
VStack(alignment: .center, spacing: 20) {
Text("No results found for this query")
.font(Font.system(size: 18, weight: .bold))
if viewModel.showLoadingBar == .none {
VStack(alignment: .center, spacing: 20) {
Text("No results found for this query")
.font(Font.system(size: 18, weight: .bold))
}
.frame(minHeight: 400)
.frame(maxWidth: .infinity)
.padding()
}
.frame(minHeight: 400)
.frame(maxWidth: .infinity)
.padding()
Spacer()
})
@ -322,14 +324,6 @@ struct AnimatingCellHeight: AnimatableModifier {
await viewModel.loadNewItems(dataService: dataService)
}
}
.onReceive(NotificationCenter.default.publisher(for: Notification.Name("PushJSONArticle"))) { notification in
guard let jsonArticle = notification.userInfo?["article"] as? JSONArticle else { return }
guard let objectID = dataService.persist(jsonArticle: jsonArticle) else { return }
guard let linkedItem = dataService.viewContext.object(with: objectID) as? Models.LibraryItem else { return }
viewModel.pushFeedItem(item: linkedItem)
viewModel.selectedItem = linkedItem
viewModel.linkIsActive = true
}
.sheet(isPresented: $searchPresented) {
LibrarySearchView(homeFeedViewModel: self.viewModel)
}
@ -493,7 +487,8 @@ struct AnimatingCellHeight: AnimatableModifier {
options: PresentationLinkTransition.SlideTransitionOptions(edge: .trailing,
options:
PresentationLinkTransition.Options(
modalPresentationCapturesStatusBarAppearance: true
modalPresentationCapturesStatusBarAppearance: true,
preferredPresentationBackgroundColor: ThemeManager.currentBgColor
))),
isPresented: $viewModel.presentWebContainer,
destination: {
@ -715,15 +710,6 @@ struct AnimatingCellHeight: AnimatableModifier {
}
}
var redactedItems: some View {
ForEach(Array(fakeLibraryItems(dataService: dataService).enumerated()), id: \.1.id) { _, item in
let horizontalInset = CGFloat(UIDevice.isIPad ? 20 : 10)
LibraryItemCard(item: item, viewer: dataService.currentViewer)
.listRowSeparatorTint(Color.thBorderColor)
.listRowInsets(.init(top: 0, leading: horizontalInset, bottom: 10, trailing: horizontalInset))
}.redacted(reason: .placeholder)
}
var listItems: some View {
ForEach(Array(viewModel.fetcher.items.enumerated()), id: \.1.unwrappedID) { idx, item in
let horizontalInset = CGFloat(UIDevice.isIPad ? 20 : 10)
@ -814,9 +800,7 @@ struct AnimatingCellHeight: AnimatableModifier {
}
}
if viewModel.showLoadingBar == .redacted {
redactedItems
} else if viewModel.showLoadingBar == .simple {
if viewModel.showLoadingBar == .redacted || viewModel.showLoadingBar == .simple {
VStack {
ProgressView()
}
@ -843,7 +827,9 @@ struct AnimatingCellHeight: AnimatableModifier {
}, header: {
filtersHeader
})
BottomView(viewModel: viewModel)
if viewModel.showLoadingBar == .none {
BottomView(viewModel: viewModel)
}
}
.padding(0)
.listStyle(.plain)
@ -1001,14 +987,7 @@ struct AnimatingCellHeight: AnimatableModifier {
ScrollView {
LazyVGrid(columns: [GridItem(.adaptive(minimum: 325, maximum: 400), spacing: 16)], alignment: .center, spacing: 30) {
if viewModel.showLoadingBar == .redacted {
ForEach(fakeLibraryItems(dataService: dataService), id: \.id) { item in
GridCard(item: item)
.aspectRatio(1.0, contentMode: .fill)
.background(Color.systemBackground)
.cornerRadius(6)
}.redacted(reason: .placeholder)
} else if viewModel.showLoadingBar == .simple {
if viewModel.showLoadingBar == .redacted || viewModel.showLoadingBar == .simple {
VStack {
ProgressView()
}
@ -1056,7 +1035,7 @@ struct AnimatingCellHeight: AnimatableModifier {
}
}
if viewModel.fetcher.items.isEmpty {
if viewModel.fetcher.items.isEmpty || viewModel.showLoadingBar == .redacted || viewModel.showLoadingBar == .simple {
EmptyState(viewModel: viewModel)
} else {
HStack {
@ -1125,31 +1104,6 @@ struct LinkDestination: View {
}
}
func fakeLibraryItems(dataService _: DataService) -> [LibraryItemData] {
Array(
repeatElement(0, count: 20)
.map { _ in
LibraryItemData(
id: UUID().uuidString,
title: "fake title that is kind of long so it looks better",
pageURLString: "",
isArchived: false,
author: "fake author",
deepLink: nil,
hasLabels: false,
noteText: nil,
readingProgress: 10,
wordsCount: 10,
isPDF: false,
highlights: nil,
sortedLabels: [],
imageURL: nil,
publisherDisplayName: "fake publisher",
descriptionText: "This is a fake description"
)
})
}
struct BottomView: View {
@ObservedObject var viewModel: HomeFeedViewModel
@EnvironmentObject var dataService: DataService

10
apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift
View file

@ -73,7 +73,12 @@ enum LoadingBarStyle {
self.linkIsActive = true
}
}
func pushLinkedRequest(request: LinkRequest) {
self.linkRequest = request
self.presentWebContainer = true
}
private var filterState: FetcherFilterState? {
if let appliedFilter = appliedFilter {
return FetcherFilterState(
@ -245,8 +250,7 @@ enum LoadingBarStyle {
}
func setLinkArchived(dataService: DataService, objectID: NSManagedObjectID, archived: Bool) {
dataService.archiveLink(objectID: objectID, archived: archived)
snackbar(archived ? "Link archived" : "Link unarchived")
archiveLibraryItemAction(dataService: dataService, objectID: objectID, archived: archived)
}
func removeLibraryItem(dataService: DataService, objectID: NSManagedObjectID) {

1
apple/OmnivoreKit/Sources/App/Views/Home/LibraryAddFeedView.swift
View file

@ -1,4 +1,3 @@
import Introspect
import Models
import Services

1
apple/OmnivoreKit/Sources/App/Views/Home/LibraryAddLinkView.swift
View file

@ -1,4 +1,3 @@
import Introspect
import Models
import Services

3
apple/OmnivoreKit/Sources/App/Views/Home/LibrarySearchView.swift
View file

@ -101,7 +101,8 @@
Spacer()
Image(systemName: "chevron.right")
}.onTapGesture {
viewModel.linkRequest = LinkRequest(id: UUID(), serverID: item.id)
homeFeedViewModel.pushLinkedRequest(request: LinkRequest(id: UUID(), serverID: item.id))
dismiss()
}
}
}

2
apple/OmnivoreKit/Sources/App/Views/InformationalSnackBar.swift
View file

@ -12,7 +12,7 @@ struct InformationalSnackbar: View {
Text(message)
}
Spacer()
if let undoAction = self.undoAction {
Button(action: {
undoAction()

1
apple/OmnivoreKit/Sources/App/Views/Labels/ApplyLabelsView.swift
View file

@ -1,4 +1,3 @@
import Models
import Services
import SwiftUI

4
apple/OmnivoreKit/Sources/App/Views/LibrarySplitView.swift
View file

@ -48,6 +48,10 @@ public struct LibrarySplitView: View {
$0.preferredPrimaryColumnWidth = 230
$0.displayModeButtonVisibility = .always
}
.onReceive(NotificationCenter.default.publisher(for: Notification.Name("PushLibraryItem"))) { notification in
guard let libraryItemId = notification.userInfo?["libraryItemId"] as? String else { return }
viewModel.pushLinkedRequest(request: LinkRequest(id: UUID(), serverID: libraryItemId))
}
.onOpenURL { url in
viewModel.linkRequest = nil

35
apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift
View file

@ -23,6 +23,8 @@ struct LibraryTabView: View {
@State var isEditMode: EditMode = .inactive
@State var showExpandedAudioPlayer = false
@State var presentPushContainer = true
@State var pushLinkRequest: String?
private let syncManager = LibrarySyncManager()
@ -77,13 +79,34 @@ struct LibraryTabView: View {
var body: some View {
VStack(spacing: 0) {
WindowLink(level: .alert, transition: .move(edge: .bottom), isPresented: $showOperationToast) {
OperationToast(operationMessage: $operationMessage,
OperationToast(operationMessage: $operationMessage,
showOperationToast: $showOperationToast,
operationStatus: $operationStatus)
} label: {
EmptyView()
}.buttonStyle(.plain)
if let pushLinkRequest = pushLinkRequest {
PresentationLink(
transition: PresentationLinkTransition.slide(
options: PresentationLinkTransition.SlideTransitionOptions(
edge: .trailing,
options: PresentationLinkTransition.Options(
modalPresentationCapturesStatusBarAppearance: true,
preferredPresentationBackgroundColor: ThemeManager.currentBgColor
))),
isPresented: $presentPushContainer,
destination: {
WebReaderLoadingContainer(requestID: pushLinkRequest)
.background(ThemeManager.currentBgColor)
.environmentObject(dataService)
.environmentObject(audioController)
}, label: {
EmptyView()
}
)
}
TabView(selection: $selectedTab) {
if !hideFollowingTab {
NavigationView {
@ -144,6 +167,11 @@ struct LibraryTabView: View {
await syncManager.syncUpdates(dataService: dataService)
}
}
.onReceive(NotificationCenter.default.publisher(for: Notification.Name("PushLibraryItem"))) { notification in
guard let libraryItemId = notification.userInfo?["libraryItemId"] as? String else { return }
pushLinkRequest = libraryItemId
presentPushContainer = true
}
.onOpenURL { url in
inboxViewModel.linkRequest = nil
@ -162,10 +190,7 @@ struct LibraryTabView: View {
case let .webAppLinkRequest(requestID):
DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(100)) {
withoutAnimation {
inboxViewModel.linkRequest = LinkRequest(id: UUID(), serverID: requestID)
inboxViewModel.presentWebContainer = true
}
inboxViewModel.pushLinkedRequest(request: LinkRequest(id: UUID(), serverID: requestID))
}
}
}

26
apple/OmnivoreKit/Sources/App/Views/Profile/ProfileView.swift
View file

@ -136,6 +136,9 @@ struct ProfileView: View {
#if os(iOS)
Section {
NavigationLink(destination: ReaderSettingsView()) {
Text(LocalText.readerSettingsGeneric)
}
NavigationLink(destination: PushNotificationSettingsView()) {
Text(LocalText.pushNotificationsGeneric)
}
@ -155,12 +158,21 @@ struct ProfileView: View {
label: { Text(LocalText.documentationGeneric) }
)
#if os(iOS)
Button(
action: { DataService.showIntercomMessenger?() },
label: { Text(LocalText.feedbackGeneric) }
)
#endif
#if os(iOS)
Button(
action: { DataService.showIntercomMessenger?() },
label: { Text(LocalText.feedbackGeneric) }
)
#endif
Button(
action: {
if let url = URL(string: "https://apps.apple.com/app/id1564031042?action=write-review") {
openURL(url)
}
},
label: { Text("Review Omnivore") }
)
Button(
action: {
@ -170,7 +182,9 @@ struct ProfileView: View {
},
label: { Text("Join community on Discord") }
)
}
Section {
Button(
action: {
if let url = URL(string: "https://omnivore.app/privacy") {

170
apple/OmnivoreKit/Sources/App/Views/Profile/PushNotificationSettingsView.swift
View file

@ -4,47 +4,121 @@
import SwiftUI
import Utils
import Views
import Transmission
@MainActor final class PushNotificationSettingsViewModel: ObservableObject {
@Published var isLoading = false
@Published var emails = [NewsletterEmail]()
@Published var desiredNotificationsEnabled = false
@AppStorage(UserDefaultKey.notificationsEnabled.rawValue) var notificationsEnabled = false
@MainActor final class PushNotificationSettingsViewModel: ObservableObject {
@Published var isLoading = false
@Published var isLoadingRules = true
func checkPushNotificationsStatus() {
UNUserNotificationCenter.current().getNotificationSettings { settings in
DispatchQueue.main.async {
self.desiredNotificationsEnabled = settings.alertSetting == UNNotificationSetting.enabled
}
@Published var emails = [NewsletterEmail]()
@Published var desiredNotificationsEnabled = false
@Published var allSubscriptionsNotificationRule: Rule?
@Published var hasSubscriptionsNotifyRule = false
@Published var showOperationToast = false
@Published var operationStatus: OperationStatus = .none
@Published var operationMessage: String?
let subscriptionRuleName = "system.autoNotify.subscriptions"
@AppStorage(UserDefaultKey.notificationsEnabled.rawValue) var notificationsEnabled = false
func checkPushNotificationsStatus() {
UNUserNotificationCenter.current().getNotificationSettings { settings in
DispatchQueue.main.async {
let desired = UserDefaults.standard.bool(forKey: UserDefaultKey.notificationsEnabled.rawValue)
self.desiredNotificationsEnabled = desired && settings.alertSetting == UNNotificationSetting.enabled
}
}
}
func tryUpdateToDesired(dataService: DataService) {
UserDefaults.standard.set(desiredNotificationsEnabled, forKey: UserDefaultKey.notificationsEnabled.rawValue)
func tryUpdateToDesired(dataService: DataService) {
UserDefaults.standard.set(desiredNotificationsEnabled, forKey: UserDefaultKey.notificationsEnabled.rawValue)
if desiredNotificationsEnabled {
UNUserNotificationCenter.current().requestAuthorization(options: [.alert]) { granted, _ in
DispatchQueue.main.async {
self.desiredNotificationsEnabled = granted
Task {
if let savedToken = UserDefaults.standard.string(forKey: UserDefaultKey.firebasePushToken.rawValue) {
_ = try? await dataService.syncDeviceToken(
deviceTokenOperation: DeviceTokenOperation.addToken(token: savedToken))
}
NotificationCenter.default.post(name: Notification.Name("ReconfigurePushNotifications"), object: nil)
if desiredNotificationsEnabled {
UNUserNotificationCenter.current().requestAuthorization(options: [.alert]) { granted, _ in
DispatchQueue.main.async {
self.desiredNotificationsEnabled = granted
Task {
if let savedToken = UserDefaults.standard.string(forKey: UserDefaultKey.firebasePushToken.rawValue) {
_ = try? await dataService.syncDeviceToken(
deviceTokenOperation: DeviceTokenOperation.addToken(token: savedToken))
}
NotificationCenter.default.post(name: Notification.Name("ReconfigurePushNotifications"), object: nil)
}
}
} else {
if let tokenID = UserDefaults.standard.string(forKey: UserDefaultKey.deviceTokenID.rawValue) {
Task {
try? await Services().dataService.syncDeviceToken(deviceTokenOperation: .deleteToken(tokenID: tokenID))
}
}
} else {
if let tokenID = UserDefaults.standard.string(forKey: UserDefaultKey.deviceTokenID.rawValue) {
Task {
try? await Services().dataService.syncDeviceToken(deviceTokenOperation: .deleteToken(tokenID: tokenID))
}
}
}
}
func loadRule(dataService: DataService) async {
do {
let rule = try await dataService.rules().filter { $0.name == subscriptionRuleName }.first
setAllSubscriptionRule(rule: rule)
} catch {
print("error fetching", error)
setAllSubscriptionRule(rule: nil)
}
}
func setAllSubscriptionRule(rule: Rule?) {
allSubscriptionsNotificationRule = rule
hasSubscriptionsNotifyRule = allSubscriptionsNotificationRule != nil
isLoadingRules = false
}
func createSubscriptionNotificationRule(dataService: DataService) {
if allSubscriptionsNotificationRule != nil {
return
}
operationMessage = "Creating notification rule..."
operationStatus = .isPerforming
showOperationToast = true
Task {
do {
let rule = try await dataService.createNotificationRule(
name: subscriptionRuleName,
filter: "in:all has:subscription"
)
setAllSubscriptionRule(rule: rule)
operationMessage = "Rule created"
operationStatus = .success
} catch {
print("error creating notification rule: ", error)
operationMessage = "Failed to create notification rule"
operationStatus = .failure
}
}
}
func deleteSubscriptionNotificationRule(dataService: DataService) {
operationMessage = "Creating label rule..."
operationStatus = .isPerforming
showOperationToast = true
Task {
do {
if let allSubscriptionsNotificationRule = allSubscriptionsNotificationRule {
_ = try await dataService.deleteRule(ruleID: allSubscriptionsNotificationRule.id)
setAllSubscriptionRule(rule: nil)
operationMessage = "Notification rule deleted"
operationStatus = .success
}
} catch {
operationMessage = "Failed to create label rule"
operationStatus = .failure
}
}
}
}
struct PushNotificationSettingsView: View {
@EnvironmentObject var dataService: DataService
@Environment(\.dismiss) private var dismiss
@ -54,6 +128,12 @@
var body: some View {
Group {
WindowLink(level: .alert, transition: .move(edge: .bottom), isPresented: $viewModel.showOperationToast) {
OperationToast(operationMessage: $viewModel.operationMessage, showOperationToast: $viewModel.showOperationToast, operationStatus: $viewModel.operationStatus)
} label: {
EmptyView()
}.buttonStyle(.plain)
#if os(iOS)
Form {
innerBody
@ -68,7 +148,10 @@
.onReceive(NotificationCenter.default.publisher(for: Notification.Name("ScrollToTop"))) { _ in
dismiss()
}
.task { viewModel.checkPushNotificationsStatus() }
.task {
viewModel.checkPushNotificationsStatus()
await viewModel.loadRule(dataService: dataService)
}
}
private var notificationsText: some View {
@ -84,6 +167,14 @@
.accentColor(.blue)
}
private var rulesSection: some View {
if viewModel.isLoadingRules {
AnyView(EmptyView())
} else {
AnyView(Toggle("Notify me when new items arrive from my subscriptions", isOn: $viewModel.hasSubscriptionsNotifyRule))
}
}
private var innerBody: some View {
Group {
Section {
@ -96,12 +187,35 @@
notificationsText
}
Section {
rulesSection
}
Section {
NavigationLink("Devices") {
PushNotificationDevicesView()
}
}
}
.onChange(of: viewModel.hasSubscriptionsNotifyRule) { newValue in
print("has notification rule: \(newValue)")
if viewModel.isLoadingRules {
return
}
if newValue {
viewModel.createSubscriptionNotificationRule(dataService: dataService)
} else {
viewModel.deleteSubscriptionNotificationRule(dataService: dataService)
}
}
.onChange(of: viewModel.operationStatus) { newValue in
if newValue == .success || newValue == .failure {
DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(1000)) {
viewModel.showOperationToast = false
}
}
}
.navigationTitle(LocalText.pushNotificationsGeneric)
}
}

27
apple/OmnivoreKit/Sources/App/Views/Profile/ReaderSettingsView.swift Normal file
View file

@ -0,0 +1,27 @@
import Services
import SwiftUI
import Views
import Utils
enum OpenLinkIn: String {
case insideApp
case systemBrowser
}
struct ReaderSettingsView: View {
@Environment(\.dismiss) private var dismiss
@AppStorage(UserDefaultKey.openExternalLinksIn.rawValue) var openExternalLinksIn = OpenLinkIn.insideApp.rawValue
var body: some View {
List {
Picker(selection: $openExternalLinksIn, content: {
Text("Inside app").tag(OpenLinkIn.insideApp.rawValue)
Text("Use system browser").tag(OpenLinkIn.systemBrowser.rawValue)
}, label: { Text("Open links:") })
.pickerStyle(MenuPickerStyle())
}.navigationTitle(LocalText.readerSettingsGeneric)
.onReceive(NotificationCenter.default.publisher(for: Notification.Name("ScrollToTop"))) { _ in
dismiss()
}
}
}

108
apple/OmnivoreKit/Sources/App/Views/Profile/SubscriptionsView.swift
View file

@ -98,11 +98,11 @@ typealias OperationStatusHandler = (_: OperationStatus) -> Void
}
}
func updateSubscription(dataService: DataService, subscription: Subscription, folder: String? = nil, fetchContent: Bool? = nil) async {
func updateSubscription(dataService: DataService, subscription: Subscription, folder: String? = nil, fetchContentType: FetchContentType? = nil) async {
operationMessage = "Updating subscription..."
operationStatus = .isPerforming
do {
try await dataService.updateSubscription(subscription.subscriptionID, folder: folder, fetchContent: fetchContent)
try await dataService.updateSubscription(subscription.subscriptionID, folder: folder, fetchContentType: fetchContentType)
operationMessage = "Subscription updated"
operationStatus = .success
} catch {
@ -240,23 +240,27 @@ struct SubscriptionsView: View {
#endif
}
private var emptyView: some View {
VStack(alignment: .center, spacing: 20) {
Text("You don't have any Feed items.")
.font(Font.system(size: 18, weight: .bold))
Text("Add an RSS/Atom feed")
.foregroundColor(Color.blue)
.onTapGesture {
showAddFeedView = true
}
}
.frame(minHeight: 80)
.frame(maxWidth: .infinity)
.padding()
}
private var innerBody: some View {
Group {
List {
Section("Feeds") {
if viewModel.feeds.count <= 0, !viewModel.isLoading {
VStack(alignment: .center, spacing: 20) {
Text("You don't have any Feed items.")
.font(Font.system(size: 18, weight: .bold))
Text("Add an RSS/Atom feed")
.foregroundColor(Color.blue)
.onTapGesture {
showAddFeedView = true
}
}
.frame(minHeight: 80)
.frame(maxWidth: .infinity)
.padding()
emptyView
} else {
ForEach(viewModel.feeds, id: \.subscriptionID) { subscription in
PresentationLink(transition: UIDevice.isIPad ? .popover : .sheet(detents: [.medium])) {
@ -264,7 +268,7 @@ struct SubscriptionsView: View {
subscription: subscription,
viewModel: viewModel,
dataService: dataService,
prefetchContent: subscription.fetchContent,
fetchContentType: subscription.fetchContentType,
folderSelection: subscription.folder,
unsubscribe: { _ in
viewModel.operationStatus = .isPerforming
@ -273,7 +277,7 @@ struct SubscriptionsView: View {
await viewModel.cancelSubscription(dataService: dataService, subscription: subscription)
}
}
)
).background(Color.systemBackground)
} label: {
SubscriptionCell(subscription: subscription)
}
@ -296,7 +300,7 @@ struct SubscriptionsView: View {
subscription: subscription,
viewModel: viewModel,
dataService: dataService,
prefetchContent: subscription.fetchContent,
fetchContentType: subscription.fetchContentType,
folderSelection: subscription.folder,
unsubscribe: { _ in
viewModel.operationStatus = .isPerforming
@ -389,7 +393,7 @@ struct SubscriptionSettingsView: View {
let viewModel: SubscriptionsViewModel
let dataService: DataService
@State var prefetchContent = false
@State var fetchContentType: FetchContentType
@State var deleteConfirmationShown = false
@State var showDeleteCompleted = false
@State var folderSelection: String = ""
@ -428,6 +432,28 @@ struct SubscriptionSettingsView: View {
return nil
}
var fetchContentRow: some View {
Picker(selection: $fetchContentType, content: {
Text("Always").tag(FetchContentType.always)
Text("Never").tag(FetchContentType.never)
Text("When empty").tag(FetchContentType.whenEmpty)
}, label: { Text("Fetch link") })
.pickerStyle(MenuPickerStyle())
.onChange(of: fetchContentType) { newValue in
Task {
viewModel.showOperationToast = true
await viewModel.updateSubscription(
dataService: dataService,
subscription: subscription,
fetchContentType: newValue
)
DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(1500)) {
viewModel.showOperationToast = false
}
}
}
}
var folderRow: some View {
HStack {
Picker("Destination Folder", selection: $folderSelection) {
@ -444,19 +470,6 @@ struct SubscriptionSettingsView: View {
}
}
}
.onChange(of: prefetchContent) { newValue in
Task {
viewModel.showOperationToast = true
await viewModel.updateSubscription(
dataService: dataService,
subscription: subscription,
fetchContent: newValue
)
DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(1500)) {
viewModel.showOperationToast = false
}
}
}
}
}
@ -526,6 +539,25 @@ struct SubscriptionSettingsView: View {
}
}
}
//
// var notificationRuleRow: some View {
// HStack {
// Text("Add Labels")
// Spacer()
// if isLoadingRule || viewModel.rules != nil {
// Button(action: { showLabelsSelector = true }, label: {
// if let ruleLabels = ruleLabels {
// let labelNames = ruleLabels.map(\.unwrappedName)
// Text("[\(labelNames.joined(separator: ","))]")
// } else {
// Text("Create Rule")
// }
// }).tint(Color.blue)
// } else {
// ProgressView()
// }
// }
// }
var body: some View {
VStack {
@ -551,12 +583,10 @@ struct SubscriptionSettingsView: View {
.padding(.horizontal, 15)
List {
// if subscription.type != .newsletter {
// Toggle(isOn: $prefetchContent, label: { Text("Prefetch Content:") })
// .onAppear {
// prefetchContent = subscription.fetchContent
// }
// }
if subscription.type != .newsletter {
fetchContentRow
}
folderRow
labelRuleRow

52
apple/OmnivoreKit/Sources/App/Views/RemoveLibraryItemAction.swift
View file

@ -7,18 +7,20 @@ import Utils
import Views
func removeLibraryItemAction(dataService: DataService, objectID: NSManagedObjectID) {
var localPdf: String? = nil
dataService.viewContext.performAndWait {
if let item = dataService.viewContext.object(with: objectID) as? Models.LibraryItem {
item.state = "DELETED"
try? dataService.viewContext.save()
// Delete local PDF file if it exists
if let localPdf = item.localPDF, let localPdfURL = PDFUtils.localPdfURL(filename: localPdf) {
try? FileManager.default.removeItem(at: localPdfURL)
}
localPdf = item.localPDF
}
}
if let localPdf = localPdf, let localPdfURL = PDFUtils.localPdfURL(filename: localPdf) {
try? FileManager.default.removeItem(at: localPdfURL)
}
let syncTask = Task.detached(priority: .background) {
do {
try await Task.sleep(nanoseconds: 4_000_000_000)
@ -44,3 +46,43 @@ func removeLibraryItemAction(dataService: DataService, objectID: NSManagedObject
}
}, dismissAfter: 2000)
}
func archiveLibraryItemAction(dataService: DataService, objectID: NSManagedObjectID, archived: Bool) {
var localPdf: String? = nil
dataService.viewContext.performAndWait {
if let item = dataService.viewContext.object(with: objectID) as? Models.LibraryItem {
item.isArchived = archived
try? dataService.viewContext.save()
localPdf = item.localPDF
}
}
// Delete local PDF file if it exists
if let localPdf = localPdf, let localPdfURL = PDFUtils.localPdfURL(filename: localPdf) {
try? FileManager.default.removeItem(at: localPdfURL)
}
let syncTask = Task.detached(priority: .background) {
do {
try await Task.sleep(nanoseconds: 4_000_000_000)
let canceled = Task.isCancelled
if !canceled {
dataService.archiveLink(objectID: objectID, archived: archived)
}
} catch {
print("error running task: ", error)
}
print("checking if task is canceled: ", Task.isCancelled)
}
Snackbar.show(message: "Item archived", undoAction: {
print("canceling task", syncTask)
syncTask.cancel()
dataService.viewContext.performAndWait {
if let item = dataService.viewContext.object(with: objectID) as? Models.LibraryItem {
item.state = "SUCCEEDED"
try? dataService.viewContext.save()
}
}
}, dismissAfter: 2000)
}

4
apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift
View file

@ -66,6 +66,10 @@ struct WebReader: PlatformViewRepresentable {
webView.scrollView.verticalScrollIndicatorInsets.top = readerViewNavBarHeight
webView.configuration.userContentController.add(webView, name: "viewerAction")
if #available(iOS 15.4, *) {
webView.configuration.preferences.isElementFullscreenEnabled = true
}
webView.scrollView.indicatorStyle = ThemeManager.currentTheme.isDark ?
UIScrollView.IndicatorStyle.white :
UIScrollView.IndicatorStyle.black

32
apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift
View file

@ -405,25 +405,37 @@ struct WebReaderContainerView: View {
anchorIndex: Int(item.readingProgressAnchor),
force: false
)
Task {
await audioController.preload(itemIDs: [item.unwrappedID])
}
viewModel.trackReadEvent(item: item)
// Wait 1.5s while loading the reader before attempting to preload the speech file
DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(1500)) {
Task {
await audioController.preload(itemIDs: [item.unwrappedID])
}
}
}
.confirmationDialog(linkToOpen?.absoluteString ?? "", isPresented: $displayLinkSheet,
titleVisibility: .visible) {
Button(action: {
if let linkToOpen = linkToOpen {
safariWebLink = SafariWebLink(id: UUID(), url: linkToOpen)
if UserDefaults.standard.string(forKey: UserDefaultKey.openExternalLinksIn.rawValue) == OpenLinkIn.systemBrowser.rawValue, UIApplication.shared.canOpenURL(linkToOpen) {
UIApplication.shared.open(linkToOpen)
} else {
safariWebLink = SafariWebLink(id: UUID(), url: linkToOpen)
}
}
}, label: { Text(LocalText.genericOpen) })
Button(action: {
#if os(iOS)
UIPasteboard.general.string = item.unwrappedPageURLString
#else
// Pasteboard.general.string = item.unwrappedPageURLString TODO: fix for mac
#endif
Snackbar.show(message: "Link copied", dismissAfter: 2000)
if let linkToOpen = linkToOpen?.absoluteString {
#if os(iOS)
UIPasteboard.general.string = linkToOpen
#else
// Pasteboard.general.string = item.unwrappedPageURLString TODO: fix for mac
#endif
Snackbar.show(message: "Link copied", dismissAfter: 2000)
} else {
Snackbar.show(message: "Error copying link", dismissAfter: 2000)
}
}, label: { Text(LocalText.readerCopyLink) })
Button(action: {
if let linkToOpen = linkToOpen {

9
apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContent.swift
View file

@ -56,6 +56,15 @@ struct WebReaderContent {
<style>
@import url("highlight\(isDark ? "-dark" : "").css");
</style>
<style>
body {
-webkit-text-size-adjust: 100%;
}
.is-sticky {
right: 20px !important;
bottom: 60px !important;
}
</style>
</head>
<body>
<div id="root" />

28
apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderLoadingContainer.swift
View file

@ -10,9 +10,15 @@ import Views
@Published var errorMessage: String?
func loadItem(dataService: DataService, username: String, requestID: String) async {
if let cached = Models.LibraryItem.lookup(byID: requestID, inContext: dataService.viewContext) {
item = cached
return
}
guard let objectID = try? await dataService.loadItemContentUsingRequestID(username: username,
requestID: requestID)
else {
errorMessage = "Item is no longer available"
return
}
item = dataService.viewContext.object(with: objectID) as? Models.LibraryItem
@ -57,8 +63,6 @@ public struct WebReaderLoadingContainer: View {
PDFWrapperView(pdfURL: pdfURL)
}
#endif
} else if item.state == "CONTENT_NOT_FETCHED" {
ProgressView()
} else {
WebReaderContainerView(item: item)
#if os(iOS)
@ -68,15 +72,23 @@ public struct WebReaderLoadingContainer: View {
.onAppear { viewModel.trackReadEvent() }
}
} else if let errorMessage = viewModel.errorMessage {
Text(errorMessage)
NavigationView {
VStack(spacing: 15) {
Text(errorMessage)
Button(action: {
dismiss()
}, label: {
Text("Dismiss")
})
}
}
#if os(iOS)
.navigationViewStyle(.stack)
#endif
} else {
ProgressView()
.task {
if let username = dataService.currentViewer?.username {
await viewModel.loadItem(dataService: dataService, username: username, requestID: requestID)
} else {
viewModel.errorMessage = "You are not logged in."
}
await viewModel.loadItem(dataService: dataService, username: "me", requestID: requestID)
}
}
}

21
apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift
View file

@ -50,27 +50,6 @@ public struct LinkedItemAudioProperties {
public let startOffset: Double
}
// Internal model used for parsing a push notification object only
public struct JSONArticle: Decodable {
public let id: String
public let title: String
public let createdAt: Date
public let updatedAt: Date
public let savedAt: Date
public let readAt: Date?
public let folder: String
public let image: String
public let readingProgressPercent: Double
public let readingProgressAnchorIndex: Int
public let slug: String
public let contentReader: String
public let url: String
public let isArchived: Bool
public let language: String?
public let wordsCount: Int?
public let downloadURL: String
}
public extension LibraryItem {
var unwrappedID: String { id ?? "" }
var unwrappedSlug: String { slug ?? "" }

1
apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift
View file

@ -1,6 +1,5 @@
import CoreData
import Foundation
import Models
import Utils
public struct PDFItem {

12
apple/OmnivoreKit/Sources/Models/DataModels/Subscription.swift
View file

@ -8,7 +8,7 @@ public struct Subscription {
public let name: String
public let type: SubscriptionType
public let folder: String
public let fetchContent: Bool
public let fetchContentType: FetchContentType
public let newsletterEmailAddress: String?
public let status: SubscriptionStatus
public let unsubscribeHttpUrl: String?
@ -24,7 +24,7 @@ public struct Subscription {
name: String,
type: SubscriptionType,
folder: String,
fetchContent: Bool,
fetchContentType: FetchContentType,
newsletterEmailAddress: String?,
status: SubscriptionStatus,
unsubscribeHttpUrl: String?,
@ -39,7 +39,7 @@ public struct Subscription {
self.name = name
self.type = type
self.folder = folder
self.fetchContent = fetchContent
self.fetchContentType = fetchContentType
self.newsletterEmailAddress = newsletterEmailAddress
self.status = status
self.unsubscribeHttpUrl = unsubscribeHttpUrl
@ -60,3 +60,9 @@ public enum SubscriptionType {
case newsletter
case feed
}
public enum FetchContentType {
case always
case never
case whenEmpty
}

267
apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift
View file

@ -667,6 +667,8 @@ extension Objects {
let contentReader: [String: Enums.ContentReader]
let createdAt: [String: DateTime]
let description: [String: String]
let directionality: [String: Enums.DirectionalityType]
let feedContent: [String: String]
let folder: [String: String]
let hasContent: [String: Bool]
let hash: [String: String]
@ -742,6 +744,14 @@ extension Objects.Article: Decodable {
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "directionality":
if let value = try container.decode(Enums.DirectionalityType?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "feedContent":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "folder":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
@ -901,6 +911,8 @@ extension Objects.Article: Decodable {
contentReader = map["contentReader"]
createdAt = map["createdAt"]
description = map["description"]
directionality = map["directionality"]
feedContent = map["feedContent"]
folder = map["folder"]
hasContent = map["hasContent"]
hash = map["hash"]
@ -1025,6 +1037,36 @@ extension Fields where TypeLock == Objects.Article {
}
}
func directionality() throws -> Enums.DirectionalityType? {
let field = GraphQLField.leaf(
name: "directionality",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
return data.directionality[field.alias!]
case .mocking:
return nil
}
}
func feedContent() throws -> String? {
let field = GraphQLField.leaf(
name: "feedContent",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
return data.feedContent[field.alias!]
case .mocking:
return nil
}
}
func folder() throws -> String {
let field = GraphQLField.leaf(
name: "folder",
@ -8127,6 +8169,7 @@ extension Objects {
let quote: [String: String]
let reactions: [String: [Objects.Reaction]]
let replies: [String: [Objects.HighlightReply]]
let representation: [String: Enums.RepresentationType]
let sharedAt: [String: DateTime]
let shortId: [String: String]
let suffix: [String: String]
@ -8208,6 +8251,10 @@ extension Objects.Highlight: Decodable {
if let value = try container.decode([Objects.HighlightReply]?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "representation":
if let value = try container.decode(Enums.RepresentationType?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "sharedAt":
if let value = try container.decode(DateTime?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
@ -8256,6 +8303,7 @@ extension Objects.Highlight: Decodable {
quote = map["quote"]
reactions = map["reactions"]
replies = map["replies"]
representation = map["representation"]
sharedAt = map["sharedAt"]
shortId = map["shortId"]
suffix = map["suffix"]
@ -8494,6 +8542,24 @@ extension Fields where TypeLock == Objects.Highlight {
}
}
func representation() throws -> Enums.RepresentationType {
let field = GraphQLField.leaf(
name: "representation",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
if let data = data.representation[field.alias!] {
return data
}
throw HttpError.badpayload
case .mocking:
return Enums.RepresentationType.allCases.first!
}
}
func sharedAt() throws -> DateTime? {
let field = GraphQLField.leaf(
name: "sharedAt",
@ -8985,6 +9051,7 @@ extension Objects {
let enabled: [String: Bool]
let id: [String: String]
let name: [String: String]
let settings: [String: String]
let taskName: [String: String]
let token: [String: String]
let type: [String: Enums.IntegrationType]
@ -9024,6 +9091,10 @@ extension Objects.Integration: Decodable {
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "settings":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "taskName":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
@ -9054,6 +9125,7 @@ extension Objects.Integration: Decodable {
enabled = map["enabled"]
id = map["id"]
name = map["name"]
settings = map["settings"]
taskName = map["taskName"]
token = map["token"]
type = map["type"]
@ -9134,6 +9206,21 @@ extension Fields where TypeLock == Objects.Integration {
}
}
func settings() throws -> String? {
let field = GraphQLField.leaf(
name: "settings",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
return data.settings[field.alias!]
case .mocking:
return nil
}
}
func taskName() throws -> String? {
let field = GraphQLField.leaf(
name: "taskName",
@ -18126,6 +18213,7 @@ extension Selection where TypeLock == Never, Type == Never {
extension Objects {
struct SearchItem {
let __typename: TypeName = .searchItem
let aiSummary: [String: String]
let annotation: [String: String]
let archivedAt: [String: DateTime]
let author: [String: String]
@ -18134,6 +18222,8 @@ extension Objects {
let contentReader: [String: Enums.ContentReader]
let createdAt: [String: DateTime]
let description: [String: String]
let directionality: [String: Enums.DirectionalityType]
let feedContent: [String: String]
let folder: [String: String]
let highlights: [String: [Objects.Highlight]]
let id: [String: String]
@ -18146,7 +18236,6 @@ extension Objects {
let ownedByViewer: [String: Bool]
let pageId: [String: String]
let pageType: [String: Enums.PageType]
let previewContent: [String: String]
let previewContentType: [String: String]
let publishedAt: [String: DateTime]
let quote: [String: String]
@ -18188,6 +18277,10 @@ extension Objects.SearchItem: Decodable {
let field = GraphQLField.getFieldNameFromAlias(alias)
switch field {
case "aiSummary":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "annotation":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
@ -18220,6 +18313,14 @@ extension Objects.SearchItem: Decodable {
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "directionality":
if let value = try container.decode(Enums.DirectionalityType?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "feedContent":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "folder":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
@ -18268,10 +18369,6 @@ extension Objects.SearchItem: Decodable {
if let value = try container.decode(Enums.PageType?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "previewContent":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "previewContentType":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
@ -18370,6 +18467,7 @@ extension Objects.SearchItem: Decodable {
}
}
aiSummary = map["aiSummary"]
annotation = map["annotation"]
archivedAt = map["archivedAt"]
author = map["author"]
@ -18378,6 +18476,8 @@ extension Objects.SearchItem: Decodable {
contentReader = map["contentReader"]
createdAt = map["createdAt"]
description = map["description"]
directionality = map["directionality"]
feedContent = map["feedContent"]
folder = map["folder"]
highlights = map["highlights"]
id = map["id"]
@ -18390,7 +18490,6 @@ extension Objects.SearchItem: Decodable {
ownedByViewer = map["ownedByViewer"]
pageId = map["pageId"]
pageType = map["pageType"]
previewContent = map["previewContent"]
previewContentType = map["previewContentType"]
publishedAt = map["publishedAt"]
quote = map["quote"]
@ -18417,6 +18516,21 @@ extension Objects.SearchItem: Decodable {
}
extension Fields where TypeLock == Objects.SearchItem {
func aiSummary() throws -> String? {
let field = GraphQLField.leaf(
name: "aiSummary",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
return data.aiSummary[field.alias!]
case .mocking:
return nil
}
}
func annotation() throws -> String? {
let field = GraphQLField.leaf(
name: "annotation",
@ -18543,6 +18657,36 @@ extension Fields where TypeLock == Objects.SearchItem {
}
}
func directionality() throws -> Enums.DirectionalityType? {
let field = GraphQLField.leaf(
name: "directionality",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
return data.directionality[field.alias!]
case .mocking:
return nil
}
}
func feedContent() throws -> String? {
let field = GraphQLField.leaf(
name: "feedContent",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
return data.feedContent[field.alias!]
case .mocking:
return nil
}
}
func folder() throws -> String {
let field = GraphQLField.leaf(
name: "folder",
@ -18737,21 +18881,6 @@ extension Fields where TypeLock == Objects.SearchItem {
}
}
func previewContent() throws -> String? {
let field = GraphQLField.leaf(
name: "previewContent",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
return data.previewContent[field.alias!]
case .mocking:
return nil
}
}
func previewContentType() throws -> String? {
let field = GraphQLField.leaf(
name: "previewContentType",
@ -21284,6 +21413,7 @@ extension Objects {
let description: [String: String]
let failedAt: [String: DateTime]
let fetchContent: [String: Bool]
let fetchContentType: [String: Enums.FetchContentType]
let folder: [String: String]
let icon: [String: String]
let id: [String: String]
@ -21342,6 +21472,10 @@ extension Objects.Subscription: Decodable {
if let value = try container.decode(Bool?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "fetchContentType":
if let value = try container.decode(Enums.FetchContentType?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "folder":
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
@ -21418,6 +21552,7 @@ extension Objects.Subscription: Decodable {
description = map["description"]
failedAt = map["failedAt"]
fetchContent = map["fetchContent"]
fetchContentType = map["fetchContentType"]
folder = map["folder"]
icon = map["icon"]
id = map["id"]
@ -21536,6 +21671,24 @@ extension Fields where TypeLock == Objects.Subscription {
}
}
func fetchContentType() throws -> Enums.FetchContentType {
let field = GraphQLField.leaf(
name: "fetchContentType",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
if let data = data.fetchContentType[field.alias!] {
return data
}
throw HttpError.badpayload
case .mocking:
return Enums.FetchContentType.allCases.first!
}
}
func folder() throws -> String {
let field = GraphQLField.leaf(
name: "folder",
@ -24692,6 +24845,7 @@ extension Objects {
struct User {
let __typename: TypeName = .user
let email: [String: String]
let features: [String: [String?]]
let followersCount: [String: Int]
let friendsCount: [String: Int]
let id: [String: String]
@ -24730,6 +24884,10 @@ extension Objects.User: Decodable {
if let value = try container.decode(String?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "features":
if let value = try container.decode([String?]?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
}
case "followersCount":
if let value = try container.decode(Int?.self, forKey: codingKey) {
map.set(key: field, hash: alias, value: value as Any)
@ -24801,6 +24959,7 @@ extension Objects.User: Decodable {
}
email = map["email"]
features = map["features"]
followersCount = map["followersCount"]
friendsCount = map["friendsCount"]
id = map["id"]
@ -24835,6 +24994,21 @@ extension Fields where TypeLock == Objects.User {
}
}
func features() throws -> [String?]? {
let field = GraphQLField.leaf(
name: "features",
arguments: []
)
select(field)
switch response {
case let .decoding(data):
return data.features[field.alias!]
case .mocking:
return nil
}
}
func followersCount() throws -> Int? {
let field = GraphQLField.leaf(
name: "followersCount",
@ -34148,6 +34322,15 @@ extension Enums {
}
}
extension Enums {
/// DirectionalityType
enum DirectionalityType: String, CaseIterable, Codable {
case ltr = "LTR"
case rtl = "RTL"
}
}
extension Enums {
/// EmptyTrashErrorCode
enum EmptyTrashErrorCode: String, CaseIterable, Codable {
@ -34180,6 +34363,17 @@ extension Enums {
}
}
extension Enums {
/// FetchContentType
enum FetchContentType: String, CaseIterable, Codable {
case always = "ALWAYS"
case never = "NEVER"
case whenEmpty = "WHEN_EMPTY"
}
}
extension Enums {
/// FiltersErrorCode
enum FiltersErrorCode: String, CaseIterable, Codable {
@ -34521,6 +34715,15 @@ extension Enums {
}
}
extension Enums {
/// RepresentationType
enum RepresentationType: String, CaseIterable, Codable {
case content = "CONTENT"
case feedContent = "FEED_CONTENT"
}
}
extension Enums {
/// RevokeApiKeyErrorCode
enum RevokeApiKeyErrorCode: String, CaseIterable, Codable {
@ -34539,6 +34742,8 @@ extension Enums {
case archive = "ARCHIVE"
case delete = "DELETE"
case markAsRead = "MARK_AS_READ"
case sendNotification = "SEND_NOTIFICATION"
@ -35306,6 +35511,8 @@ extension InputObjects {
var quote: OptionalArgument<String> = .absent()
var representation: OptionalArgument<Enums.RepresentationType> = .absent()
var sharedAt: OptionalArgument<DateTime> = .absent()
var shortId: String
@ -35326,6 +35533,7 @@ extension InputObjects {
if patch.hasValue { try container.encode(patch, forKey: .patch) }
if prefix.hasValue { try container.encode(prefix, forKey: .prefix) }
if quote.hasValue { try container.encode(quote, forKey: .quote) }
if representation.hasValue { try container.encode(representation, forKey: .representation) }
if sharedAt.hasValue { try container.encode(sharedAt, forKey: .sharedAt) }
try container.encode(shortId, forKey: .shortId)
if suffix.hasValue { try container.encode(suffix, forKey: .suffix) }
@ -35343,6 +35551,7 @@ extension InputObjects {
case patch
case prefix
case quote
case representation
case sharedAt
case shortId
case suffix
@ -35602,6 +35811,8 @@ extension InputObjects {
var quote: String
var representation: OptionalArgument<Enums.RepresentationType> = .absent()
var shortId: String
var suffix: OptionalArgument<String> = .absent()
@ -35619,6 +35830,7 @@ extension InputObjects {
try container.encode(patch, forKey: .patch)
if prefix.hasValue { try container.encode(prefix, forKey: .prefix) }
try container.encode(quote, forKey: .quote)
if representation.hasValue { try container.encode(representation, forKey: .representation) }
try container.encode(shortId, forKey: .shortId)
if suffix.hasValue { try container.encode(suffix, forKey: .suffix) }
}
@ -35635,6 +35847,7 @@ extension InputObjects {
case patch
case prefix
case quote
case representation
case shortId
case suffix
}
@ -36228,6 +36441,8 @@ extension InputObjects {
var name: String
var settings: OptionalArgument<String> = .absent()
var syncedAt: OptionalArgument<DateTime> = .absent()
var taskName: OptionalArgument<String> = .absent()
@ -36242,6 +36457,7 @@ extension InputObjects {
if id.hasValue { try container.encode(id, forKey: .id) }
if importItemState.hasValue { try container.encode(importItemState, forKey: .importItemState) }
try container.encode(name, forKey: .name)
if settings.hasValue { try container.encode(settings, forKey: .settings) }
if syncedAt.hasValue { try container.encode(syncedAt, forKey: .syncedAt) }
if taskName.hasValue { try container.encode(taskName, forKey: .taskName) }
try container.encode(token, forKey: .token)
@ -36253,6 +36469,7 @@ extension InputObjects {
case id
case importItemState
case name
case settings
case syncedAt
case taskName
case token
@ -36511,6 +36728,8 @@ extension InputObjects {
var fetchContent: OptionalArgument<Bool> = .absent()
var fetchContentType: OptionalArgument<Enums.FetchContentType> = .absent()
var folder: OptionalArgument<String> = .absent()
var isPrivate: OptionalArgument<Bool> = .absent()
@ -36523,6 +36742,7 @@ extension InputObjects {
var container = encoder.container(keyedBy: CodingKeys.self)
if autoAddToLibrary.hasValue { try container.encode(autoAddToLibrary, forKey: .autoAddToLibrary) }
if fetchContent.hasValue { try container.encode(fetchContent, forKey: .fetchContent) }
if fetchContentType.hasValue { try container.encode(fetchContentType, forKey: .fetchContentType) }
if folder.hasValue { try container.encode(folder, forKey: .folder) }
if isPrivate.hasValue { try container.encode(isPrivate, forKey: .isPrivate) }
if subscriptionType.hasValue { try container.encode(subscriptionType, forKey: .subscriptionType) }
@ -36532,6 +36752,7 @@ extension InputObjects {
enum CodingKeys: String, CodingKey {
case autoAddToLibrary
case fetchContent
case fetchContentType
case folder
case isPrivate
case subscriptionType
@ -36828,6 +37049,8 @@ extension InputObjects {
var fetchContent: OptionalArgument<Bool> = .absent()
var fetchContentType: OptionalArgument<Enums.FetchContentType> = .absent()
var folder: OptionalArgument<String> = .absent()
var id: String
@ -36852,6 +37075,7 @@ extension InputObjects {
if description.hasValue { try container.encode(description, forKey: .description) }
if failedAt.hasValue { try container.encode(failedAt, forKey: .failedAt) }
if fetchContent.hasValue { try container.encode(fetchContent, forKey: .fetchContent) }
if fetchContentType.hasValue { try container.encode(fetchContentType, forKey: .fetchContentType) }
if folder.hasValue { try container.encode(folder, forKey: .folder) }
try container.encode(id, forKey: .id)
if isPrivate.hasValue { try container.encode(isPrivate, forKey: .isPrivate) }
@ -36868,6 +37092,7 @@ extension InputObjects {
case description
case failedAt
case fetchContent
case fetchContentType
case folder
case id
case isPrivate

46
apple/OmnivoreKit/Sources/Services/DataService/Mutations/DeleteRule.swift Normal file
View file

@ -0,0 +1,46 @@
import CoreData
import Foundation
import Models
import SwiftGraphQL
public extension DataService {
func deleteRule(ruleID: String) async throws -> Rule {
enum MutationResult {
case result(rule: Rule)
case error(errorMessage: String)
}
let selection = Selection<MutationResult, Unions.DeleteRuleResult> {
try $0.on(
deleteRuleError: .init { .error(errorMessage: try $0.errorCodes().first?.rawValue ?? "Unknown Error") },
deleteRuleSuccess: .init { .result(rule: try $0.rule(selection: ruleSelection)) }
)
}
let mutation = Selection.Mutation {
try $0.deleteRule(
id: ruleID,
selection: selection
)
}
let path = appEnvironment.graphqlPath
let headers = networker.defaultHeaders
return try await withCheckedThrowingContinuation { continuation in
send(mutation, to: path, headers: headers) { queryResult in
guard let payload = try? queryResult.get() else {
continuation.resume(throwing: BasicError.message(messageText: "network error"))
return
}
switch payload.data {
case let .result(rule: rule):
continuation.resume(returning: rule)
case let .error(errorMessage: errorMessage):
continuation.resume(throwing: BasicError.message(messageText: errorMessage))
}
}
}
}
}

50
apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift
View file

@ -12,6 +12,7 @@ public struct Rule {
public enum RuleActionType {
case addLabel
case archive
case delete
case markAsRead
case sendNotification
@ -25,6 +26,8 @@ public enum RuleActionType {
return .markAsRead
case Enums.RuleActionType.sendNotification:
return .sendNotification
case .delete:
return .delete
}
}
}
@ -89,4 +92,51 @@ public extension DataService {
}
}
}
func createNotificationRule(name: String, filter: String) async throws -> Rule {
enum MutationResult {
case result(rule: Rule)
case error(errorMessage: String)
}
let selection = Selection<MutationResult, Unions.SetRuleResult> {
try $0.on(
setRuleError: .init { .error(errorMessage: try $0.errorCodes().first?.rawValue ?? "Unknown Error") },
setRuleSuccess: .init { .result(rule: try $0.rule(selection: ruleSelection)) }
)
}
let mutation = Selection.Mutation {
try $0.setRule(
input: InputObjects.SetRuleInput(
actions: [InputObjects.RuleActionInput(params: [], type: .sendNotification)],
enabled: true,
eventTypes: [.pageCreated],
filter: filter,
id: OptionalArgument(nil),
name: name
),
selection: selection
)
}
let path = appEnvironment.graphqlPath
let headers = networker.defaultHeaders
return try await withCheckedThrowingContinuation { continuation in
send(mutation, to: path, headers: headers) { queryResult in
guard let payload = try? queryResult.get() else {
continuation.resume(throwing: BasicError.message(messageText: "network error"))
return
}
switch payload.data {
case let .result(rule: rule):
continuation.resume(returning: rule)
case let .error(errorMessage: errorMessage):
continuation.resume(throwing: BasicError.message(messageText: errorMessage))
}
}
}
}
}

4
apple/OmnivoreKit/Sources/Services/DataService/Mutations/UpdateSubscription.swift
View file

@ -4,7 +4,7 @@ import Models
import SwiftGraphQL
public extension DataService {
func updateSubscription(_ subscriptionID: String, folder: String? = nil, fetchContent: Bool? = nil) async throws {
func updateSubscription(_ subscriptionID: String, folder: String? = nil, fetchContentType: FetchContentType? = nil) async throws {
enum MutationResult {
case success(subscriptionID: String)
case error(errorMessage: String)
@ -20,7 +20,7 @@ public extension DataService {
let mutation = Selection.Mutation {
try $0.updateSubscription(
input: InputObjects.UpdateSubscriptionInput(
fetchContent: OptionalArgument(fetchContent),
fetchContentType: OptionalArgument(fetchContentType?.toGQLType()),
folder: OptionalArgument(folder),
id: subscriptionID
),

25
apple/OmnivoreKit/Sources/Services/DataService/Selections/SubsciptionSelection.swift
View file

@ -11,7 +11,7 @@ let subscriptionSelection = Selection.Subscription {
name: try $0.name(),
type: try SubscriptionType.from($0.type()),
folder: try $0.folder(),
fetchContent: try $0.fetchContent(),
fetchContentType: try FetchContentType.from($0.fetchContentType()),
newsletterEmailAddress: try $0.newsletterEmail(),
status: try SubscriptionStatus.make(from: $0.status()),
unsubscribeHttpUrl: try $0.unsubscribeHttpUrl(),
@ -45,3 +45,26 @@ extension SubscriptionType {
}
}
}
extension FetchContentType {
static func from(_ other: Enums.FetchContentType) -> FetchContentType {
switch other {
case .always:
return .always
case .never:
return .never
case .whenEmpty:
return .whenEmpty
}
}
func toGQLType() -> Enums.FetchContentType {
switch self {
case .always:
return .always
case .never:
return .never
case .whenEmpty:
return .whenEmpty
}
}
}

59
apple/OmnivoreKit/Sources/Services/InternalModels/InternalLibraryItem.swift
View file

@ -126,62 +126,3 @@ extension Sequence where Element == InternalLibraryItem {
}
}
}
public extension DataService {
func persist(jsonArticle: JSONArticle) -> NSManagedObjectID? {
jsonArticle.persistAsLinkedItem(context: backgroundContext)
}
}
extension JSONArticle {
func persistAsLinkedItem(context: NSManagedObjectContext) -> NSManagedObjectID? {
var objectID: NSManagedObjectID?
let internalLinkedItem = InternalLibraryItem(
id: id,
title: title,
createdAt: createdAt,
savedAt: savedAt,
readAt: readAt,
updatedAt: updatedAt,
folder: folder,
state: .succeeded,
readingProgress: readingProgressPercent,
readingProgressAnchor: readingProgressAnchorIndex,
imageURLString: image,
onDeviceImageURLString: nil,
documentDirectoryPath: nil,
pageURLString: url,
descriptionText: title,
publisherURLString: nil,
siteName: nil,
author: nil,
publishDate: nil,
slug: slug,
isArchived: isArchived,
contentReader: contentReader,
htmlContent: nil,
originalHtml: nil,
language: language,
wordsCount: wordsCount,
downloadURL: downloadURL,
recommendations: [],
labels: [],
highlights: []
)
context.performAndWait {
objectID = internalLinkedItem.asManagedObject(inContext: context).objectID
do {
try context.save()
logger.debug("LinkedItem saved succesfully")
} catch {
context.rollback()
logger.debug("Failed to save LinkedItem: \(error.localizedDescription)")
}
}
return objectID
}
}

26
apple/OmnivoreKit/Sources/Services/NSNotification+Operation.swift
View file

@ -3,8 +3,7 @@ import Foundation
import Models
public extension NSNotification {
static let PushJSONArticle = Notification.Name("PushJSONArticle")
static let PushReaderItem = Notification.Name("PushReaderItem")
static let PushLibraryItem = Notification.Name("PushLibraryItem")
static let SnackBar = Notification.Name("SnackBar")
static let OperationFailure = Notification.Name("OperationFailure")
static let ReaderSettingsChanged = Notification.Name("ReaderSettingsChanged")
@ -19,11 +18,7 @@ public extension NSNotification {
}
static var pushFeedItemPublisher: NotificationCenter.Publisher {
NotificationCenter.default.publisher(for: PushJSONArticle)
}
static var pushReaderItemPublisher: NotificationCenter.Publisher {
NotificationCenter.default.publisher(for: PushReaderItem)
NotificationCenter.default.publisher(for: PushLibraryItem)
}
static var snackBarPublisher: NotificationCenter.Publisher {
@ -61,19 +56,14 @@ public extension NSNotification {
return nil
}
static func pushJSONArticle(article: JSONArticle) {
static func pushLibraryItem(folder: String?, libraryItemId: String) {
NotificationCenter.default.post(
name: NSNotification.PushJSONArticle,
name: NSNotification.PushLibraryItem,
object: nil,
userInfo: ["article": article]
)
}
static func pushReaderItem(objectID: NSManagedObjectID) {
NotificationCenter.default.post(
name: NSNotification.PushReaderItem,
object: nil,
userInfo: ["objectID": objectID]
userInfo: [
"folder": folder ?? "inbox",
"libraryItemId": libraryItemId
]
)
}

2
apple/OmnivoreKit/Sources/Utils/UIViewControllerExtensions.swift
View file

@ -49,6 +49,8 @@
child.didMove(toParent: self)
}
}
#endif
#if os(macOS)

1
apple/OmnivoreKit/Sources/Utils/UserDefaultKeys.swift
View file

@ -35,6 +35,7 @@ public enum UserDefaultKey: String {
case hideFeatureSection
case hideSystemLabels
case justifyText
case openExternalLinksIn
case prefersHideStatusBarInReader
case visibleShareExtensionTab
}

16
apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift
View file

@ -298,10 +298,16 @@ public final class OmnivoreWebView: WKWebView {
case #selector(removeSelection): return true
case #selector(copy(_:)): return true
case #selector(setLabels(_:)): return true
case Selector(("_lookup:")): return (currentMenu == .defaultMenu)
case Selector(("_define:")): return (currentMenu == .defaultMenu)
case Selector(("_translate:")): return (currentMenu == .defaultMenu)
case Selector(("_findSelected:")): return (currentMenu == .defaultMenu)
case Selector(("lookup:")): return (currentMenu == .defaultMenu)
case Selector(("define:")): return (currentMenu == .defaultMenu)
case Selector(("translate:")): return (currentMenu == .defaultMenu)
case Selector(("findSelected:")): return (currentMenu == .defaultMenu)
default: return false
}
}
@ -371,6 +377,14 @@ public final class OmnivoreWebView: WKWebView {
let items: [UIMenuElement]
if currentMenu == .defaultMenu {
let autoHighlightEnabled = UserDefaults.standard.value(forKey: UserDefaultKey.enableHighlightOnRelease.rawValue)
if let autoHighlightEnabled = autoHighlightEnabled as? Bool, autoHighlightEnabled {
builder.remove(menu: .standardEdit)
builder.remove(menu: .lookup)
builder.remove(menu: .find)
super.buildMenu(with: builder)
return
}
let highlight = UICommand(title: LocalText.genericHighlight, action: #selector(highlightSelection))
items = [highlight, annotate]
} else {
@ -380,7 +394,7 @@ public final class OmnivoreWebView: WKWebView {
}
let omnivore = UIMenu(title: "", options: .displayInline, children: items)
builder.insertSibling(omnivore, beforeMenu: .lookup)
builder.insertSibling(omnivore, afterMenu: .standardEdit)
}
super.buildMenu(with: builder)

1
apple/OmnivoreKit/Sources/Views/FeedItem/CardUtils.swift
View file

@ -1,4 +1,3 @@
import Foundation
func cardShouldHideUrl(_ url: String?) -> Bool {

16
apple/OmnivoreKit/Sources/Views/FeedItem/GridCard.swift
View file

@ -11,12 +11,14 @@ public enum GridCardAction {
}
public struct GridCard: View {
let item: LibraryItemData
@ObservedObject var item: Models.LibraryItem
let savedAtStr: String
public init(
item: LibraryItemData
item: Models.LibraryItem
) {
self.item = item
self.savedAtStr = savedDateString(item.savedAt)
}
var imageBox: some View {
@ -67,7 +69,7 @@ public struct GridCard: View {
var fallbackImage: some View {
GeometryReader { geo in
HStack {
Text(item.title)
Text(item.title ?? "")
.font(fallbackFont)
.frame(alignment: .center)
.multilineTextAlignment(.leading)
@ -198,6 +200,10 @@ public struct GridCard: View {
$0.icon
}
Text(savedAtStr)
.font(.footnote)
.foregroundColor(Color.themeLibraryItemSubtle)
+
Text("\(estimatedReadingTime)")
.font(.caption2).fontWeight(.medium)
.foregroundColor(Color.themeLibraryItemSubtle)
@ -232,7 +238,7 @@ public struct GridCard: View {
.dynamicTypeSize(.xSmall ... .medium)
.padding(.horizontal, 15)
Text(item.title)
Text(item.title ?? "")
.lineLimit(2)
.font(.appHeadline)
.foregroundColor(.appGrayTextContrast)
@ -246,7 +252,7 @@ public struct GridCard: View {
// Link description and image
HStack(alignment: .top) {
Text(item.descriptionText ?? item.title)
Text(item.descriptionText ?? item.title ?? "")
.font(.appSubheadline)
.foregroundColor(.appGrayTextContrast)
.lineLimit(2)

105
apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift
View file

@ -32,11 +32,11 @@ enum FlairLabels: String {
}
public extension View {
func draggableItem(item: LibraryItemData) -> some View {
func draggableItem(item: Models.LibraryItem) -> some View {
#if os(iOS)
if #available(iOS 16.0, *), let url = item.deepLink {
return AnyView(self.draggable(url) {
Label(item.title, systemImage: "link")
Label(item.title ?? "", systemImage: "link")
})
}
#endif
@ -44,77 +44,33 @@ public extension View {
}
}
public struct LibraryItemData {
public var id: String
public let title: String
public let pageURLString: String
public var isArchived: Bool
public let author: String?
public let deepLink: URL?
public let hasLabels: Bool
public let noteText: String?
public let readingProgress: Double
public let wordsCount: Int64
public let isPDF: Bool
public let highlights: NSSet?
public let sortedLabels: [LinkedItemLabel]
public let imageURL: URL?
public let publisherDisplayName: String?
public let descriptionText: String?
public init(id: String, title: String, pageURLString: String, isArchived: Bool, author: String?,
deepLink: URL?, hasLabels: Bool, noteText: String?,
readingProgress: Double, wordsCount: Int64, isPDF: Bool, highlights: NSSet?,
sortedLabels: [LinkedItemLabel], imageURL: URL?, publisherDisplayName: String?, descriptionText: String?)
{
self.id = id
self.title = title
self.pageURLString = pageURLString
self.isArchived = isArchived
self.author = author
self.deepLink = deepLink
self.hasLabels = hasLabels
self.noteText = noteText
self.readingProgress = readingProgress
self.wordsCount = wordsCount
self.isPDF = isPDF
self.highlights = highlights
self.sortedLabels = sortedLabels
self.imageURL = imageURL
self.publisherDisplayName = publisherDisplayName
self.descriptionText = descriptionText
}
public static func make(from item: Models.LibraryItem) -> LibraryItemData {
LibraryItemData(
id: item.unwrappedID,
title: item.unwrappedTitle,
pageURLString: item.unwrappedPageURLString,
isArchived: item.isArchived,
author: item.author,
deepLink: item.deepLink,
hasLabels: item.hasLabels,
noteText: item.noteText,
readingProgress: item.readingProgress,
wordsCount: item.wordsCount,
isPDF: item.isPDF,
highlights: item.highlights,
sortedLabels: item.sortedLabels,
imageURL: item.imageURL,
publisherDisplayName: item.publisherDisplayName,
descriptionText: item.descriptionText
)
func savedDateString(_ savedAt: Date?) -> String {
if let savedAt = savedAt {
let locale = Locale.current
let dateFormatter = DateFormatter()
if Calendar.current.isDateInToday(savedAt) {
dateFormatter.dateStyle = .none
dateFormatter.timeStyle = .short
} else {
dateFormatter.dateFormat = "MMM dd"
}
dateFormatter.locale = locale
return dateFormatter.string(from: savedAt) + " • "
}
return ""
}
public struct LibraryItemCard: View {
let viewer: Viewer?
var item: LibraryItemData
@ObservedObject var item: Models.LibraryItem
@State var noteLineLimit: Int? = 3
public init(item: LibraryItemData, viewer: Viewer?) {
let savedAtStr: String
public init(item: Models.LibraryItem, viewer: Viewer?) {
self.item = item
self.viewer = viewer
self.savedAtStr = savedDateString(item.savedAt)
}
public var body: some View {
@ -278,23 +234,28 @@ public struct LibraryItemCard: View {
$0.icon
}
Text(savedAtStr)
.font(.footnote)
.foregroundColor(Color.themeLibraryItemSubtle)
+
Text("\(estimatedReadingTime)")
.font(.caption2).fontWeight(.medium)
.font(.footnote)
.foregroundColor(Color.themeLibraryItemSubtle)
+
Text("\(readingProgress)")
.font(.caption2).fontWeight(.medium)
.font(.footnote)
.foregroundColor(isPartiallyRead ? Color.appGreenSuccess : Color.themeLibraryItemSubtle)
+
Text("\(highlightsText)")
.font(.caption2).fontWeight(.medium)
.font(.footnote)
.foregroundColor(Color.themeLibraryItemSubtle)
+
Text("\(notesText)")
.font(.caption2).fontWeight(.medium)
.font(.footnote)
.foregroundColor(Color.themeLibraryItemSubtle)
}
.frame(maxWidth: .infinity, alignment: .leading)
@ -344,13 +305,13 @@ public struct LibraryItemCard: View {
var byLine: some View {
if let origin = cardSiteName(item.pageURLString) {
Text(bylineStr + " | " + origin)
.font(.caption2)
.font(.footnote)
.foregroundColor(Color.themeLibraryItemSubtle)
.frame(maxWidth: .infinity, alignment: .leading)
.lineLimit(1)
} else {
Text(bylineStr)
.font(.caption2)
.font(.footnote)
.foregroundColor(Color.themeLibraryItemSubtle)
.frame(maxWidth: .infinity, alignment: .leading)
.lineLimit(1)
@ -358,11 +319,11 @@ public struct LibraryItemCard: View {
}
public var articleInfo: some View {
VStack(alignment: .leading, spacing: 5) {
VStack(alignment: .leading, spacing: 7) {
readInfo
.dynamicTypeSize(.xSmall ... .medium)
Text(item.title)
Text(item.title ?? "")
.font(.body).fontWeight(.semibold)
.lineSpacing(1.25)
.foregroundColor(.appGrayTextContrast)

1
apple/OmnivoreKit/Sources/Views/LocalText.swift
View file

@ -201,4 +201,5 @@ public enum LocalText {
public static let dismissButton = localText(key: "dismissButton")
public static let errorNetwork = localText(key: "errorNetwork")
public static let documentationGeneric = localText(key: "documentationGeneric")
public static let readerSettingsGeneric = localText(key: "readerSettingsGeneric")
}

2
apple/OmnivoreKit/Sources/Views/Resources/bundle.js
View file

File diff suppressed because one or more lines are too long

5
apple/OmnivoreKit/Sources/Views/Resources/en.lproj/Localizable.strings
View file

@ -171,7 +171,7 @@
"labelsGeneric" = "Labels";
"emailsGeneric" = "Emails";
"subscriptionsGeneric" = "Subscriptions";
"textToSpeechGeneric" = "Text to Speech";
"textToSpeechGeneric" = "Text to speech";
"privacyPolicyGeneric" = "Privacy Policy";
"termsAndConditionsGeneric" = "Terms and Conditions";
"feedbackGeneric" = "Feedback";
@ -196,7 +196,8 @@
"clubsGeneric" = "Clubs";
"filterGeneric" = "Filters";
"errorGeneric" = "Something went wrong, please try again.";
"pushNotificationsGeneric" = "Push Notifications";
"readerSettingsGeneric" = "Reader settings";
"pushNotificationsGeneric" = "Push notifications";
"dismissButton" = "Dismiss";
"errorNetwork" = "We are having trouble connecting to the internet.";
"documentationGeneric" = "Documentation";

10
apple/Sources/PushNotificationConfig.swift
View file

@ -55,7 +55,7 @@ extension AppDelegate: UNUserNotificationCenterDelegate {
let userInfo = notification.request.content.userInfo
UIApplication.shared.applicationIconBadgeNumber = 0
print(userInfo) // extract data sent along with PN
print("push data", userInfo) // extract data sent along with PN
completionHandler([[.banner, .sound]])
}
@ -68,12 +68,8 @@ extension AppDelegate: UNUserNotificationCenterDelegate {
let userInfo = response.notification.request.content.userInfo
if let linkData = userInfo["link"] as? String {
guard let jsonData = Data(base64Encoded: linkData) else { return }
if let article = try? JSONDecoder().decode(JSONArticle.self, from: jsonData) {
NSNotification.pushJSONArticle(article: article)
}
if let libraryItemId = userInfo["libraryItemId"] as? String {
NSNotification.pushLibraryItem(folder: userInfo["folder"] as? String, libraryItemId: libraryItemId)
}
completionHandler()

2
apple/Sources/SafariExtension/Resources/views/options.html
View file

@ -18,7 +18,7 @@
<p>To do this, create an API key at
<a href='https://omnivore.app/settings/api'>omnivore.app/settings/api</a>,
paste it into the textboix below, and choose 'Save API Key'.
paste it into the textbox below, and choose 'Save API Key'.
</p>
<p></p>

17
apple/Sources/ShareExtension/ShareExtensionViewController.swift
View file

@ -8,13 +8,18 @@ import Views
import UIKit
@objc(ShareExtensionViewController)
final class ShareExtensionViewController: UIViewController {
final class ShareExtensionViewController: UIViewController, UIGestureRecognizerDelegate {
let labelsViewModel = LabelsViewModel()
let viewModel = ShareExtensionViewModel()
override func viewDidLoad() {
super.viewDidLoad()
view.backgroundColor = .clear
view.backgroundColor = UIColor(white: 1.0, alpha: 0.001)
view.isUserInteractionEnabled = true
let dismissGesture = UITapGestureRecognizer(target: self, action: #selector(self.viewTapped(_:)))
dismissGesture.delegate = self
view.addGestureRecognizer(dismissGesture)
if !viewModel.services.authenticator.isLoggedIn,
!viewModel.services.dataService.appEnvironment.environmentConfigured
@ -60,6 +65,14 @@ import Views
)
}
func gestureRecognizer(_ gestureRecognizer: UIGestureRecognizer, shouldReceive touch: UITouch) -> Bool {
return touch.view == gestureRecognizer.view
}
@objc func viewTapped(_ panGesture: UIGestureRecognizer) {
viewModel.dismissExtension(extensionContext: extensionContext)
}
func openSheet(_ rootView: AnyView) {
let hostingController = UIHostingController(rootView: rootView)

2
docker-compose.yml
View file

@ -15,6 +15,8 @@ services:
retries: 3
expose:
- 5432
ports:
- "5432:5432"
migrate:
build:

15
packages/api/package.json
View file

@ -23,6 +23,8 @@
"@google-cloud/storage": "^7.0.1",
"@google-cloud/tasks": "^4.0.0",
"@graphql-tools/utils": "^9.1.1",
"@langchain/openai": "^0.0.14",
"@notionhq/client": "^2.2.14",
"@omnivore/content-handler": "1.0.0",
"@omnivore/liqe": "1.0.0",
"@omnivore/readability": "1.0.0",
@ -44,6 +46,7 @@
"@sentry/integrations": "^7.10.0",
"@sentry/node": "^5.26.0",
"@sentry/tracing": "^7.9.0",
"@types/showdown": "^2.0.6",
"addressparser": "^1.0.1",
"apollo-datasource": "^3.3.1",
"apollo-server-express": "^3.6.3",
@ -77,11 +80,13 @@
"ioredis": "^5.3.2",
"jsonwebtoken": "^8.5.1",
"jwks-rsa": "^2.0.3",
"langchain": "^0.1.21",
"linkedom": "^0.14.9",
"lodash": "^4.17.21",
"luxon": "^3.2.1",
"nanoid": "^3.1.25",
"node-html-markdown": "^1.3.0",
"node-mailjet": "^6.0.5",
"nodemailer": "^6.7.3",
"normalize-url": "^6.1.0",
"oauth": "^0.10.0",
@ -90,21 +95,26 @@
"posthog-node": "^3.6.3",
"private-ip": "^2.3.3",
"prom-client": "^15.1.0",
"rate-limit-redis": "^4.2.0",
"redis": "^4.6.13",
"rss-parser": "^3.13.0",
"sanitize-html": "^2.3.2",
"sax": "^1.3.0",
"search-query-parser": "^1.6.0",
"snake-case": "^3.0.3",
"showdown": "^2.1.0",
"snake-case": "^4.0.0",
"supertest": "^6.2.2",
"ts-loader": "^9.3.0",
"typeorm": "^0.3.4",
"typeorm-naming-strategies": "^4.1.0",
"underscore": "^1.13.6",
"url-pattern": "^1.0.3",
"urlsafe-base64": "^1.0.0",
"uuid": "^8.3.1",
"voca": "^1.4.0",
"winston": "^3.3.3",
"word-counting": "^1.1.4"
"word-counting": "^1.1.4",
"youtubei": "1.3.7"
},
"devDependencies": {
"@babel/register": "^7.14.5",
@ -133,6 +143,7 @@
"@types/private-ip": "^1.0.0",
"@types/sanitize-html": "^1.27.1",
"@types/sax": "^1.2.7",
"@types/showdown": "^2.0.6",
"@types/sinon": "^10.0.13",
"@types/sinon-chai": "^3.2.8",
"@types/supertest": "^2.0.11",

36
packages/api/src/entity/AISummary.ts Normal file
View file

@ -0,0 +1,36 @@
import {
Column,
CreateDateColumn,
Entity,
JoinColumn,
ManyToOne,
PrimaryGeneratedColumn,
} from 'typeorm'
import { User } from './user'
import { LibraryItem } from './library_item'
@Entity({ name: 'ai_summaries' })
export class AISummary {
@PrimaryGeneratedColumn('uuid')
id!: string
@ManyToOne(() => User)
@JoinColumn({ name: 'user_id' })
user!: User
@ManyToOne(() => LibraryItem)
@JoinColumn({ name: 'library_item_id' })
libraryItem!: LibraryItem
@Column('text')
summary?: string
@Column('text')
title?: string
@Column('text')
slug?: string
@CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
createdAt!: Date
}

2
packages/api/src/entity/highlight.ts
View file

@ -62,7 +62,7 @@ export class Highlight {
createdAt!: Date
@UpdateDateColumn()
updatedAt?: Date | null
updatedAt!: Date
@Column('timestamp')
sharedAt?: Date

3
packages/api/src/entity/integration.ts
View file

@ -59,4 +59,7 @@ export class Integration {
@Column('enum', { enum: ImportItemState, nullable: true })
importItemState?: ImportItemState | null
@Column('jsonb', { nullable: true })
settings?: any
}

3
packages/api/src/entity/rule.ts
View file

@ -59,4 +59,7 @@ export class Rule {
@UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
updatedAt!: Date
@Column('timestamptz')
failedAt?: Date
}

648
packages/api/src/generated/graphql.ts
View file

File diff suppressed because it is too large Load diff

253
packages/api/src/generated/schema.graphql
View file

@ -1,5 +1,26 @@
directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION
type AddDiscoverFeedError {
errorCodes: [AddDiscoverFeedErrorCode!]!
}
enum AddDiscoverFeedErrorCode {
BAD_REQUEST
CONFLICT
NOT_FOUND
UNAUTHORIZED
}
input AddDiscoverFeedInput {
url: String!
}
union AddDiscoverFeedResult = AddDiscoverFeedError | AddDiscoverFeedSuccess
type AddDiscoverFeedSuccess {
feed: DiscoverFeed!
}
type AddPopularReadError {
errorCodes: [AddPopularReadErrorCode!]!
}
@ -69,6 +90,7 @@ type Article {
contentReader: ContentReader!
createdAt: Date!
description: String
directionality: DirectionalityType
feedContent: String
folder: String!
hasContent: Boolean
@ -464,6 +486,47 @@ type DeleteAccountSuccess {
userID: ID!
}
type DeleteDiscoverArticleError {
errorCodes: [DeleteDiscoverArticleErrorCode!]!
}
enum DeleteDiscoverArticleErrorCode {
BAD_REQUEST
NOT_FOUND
UNAUTHORIZED
}
input DeleteDiscoverArticleInput {
discoverArticleId: ID!
}
union DeleteDiscoverArticleResult = DeleteDiscoverArticleError | DeleteDiscoverArticleSuccess
type DeleteDiscoverArticleSuccess {
id: ID!
}
type DeleteDiscoverFeedError {
errorCodes: [DeleteDiscoverFeedErrorCode!]!
}
enum DeleteDiscoverFeedErrorCode {
BAD_REQUEST
CONFLICT
NOT_FOUND
UNAUTHORIZED
}
input DeleteDiscoverFeedInput {
feedId: ID!
}
union DeleteDiscoverFeedResult = DeleteDiscoverFeedError | DeleteDiscoverFeedSuccess
type DeleteDiscoverFeedSuccess {
id: String!
}
type DeleteFilterError {
errorCodes: [DeleteFilterErrorCode!]!
}
@ -646,6 +709,77 @@ type DeviceTokensSuccess {
deviceTokens: [DeviceToken!]!
}
enum DirectionalityType {
LTR
RTL
}
type DiscoverFeed {
description: String
id: ID!
image: String
link: String!
title: String!
type: String!
visibleName: String
}
type DiscoverFeedArticle {
author: String
description: String!
feed: String!
id: ID!
image: String
publishedDate: Date
savedId: String
savedLinkUrl: String
siteName: String
slug: String!
title: String!
url: String!
}
type DiscoverFeedError {
errorCodes: [DiscoverFeedErrorCode!]!
}
enum DiscoverFeedErrorCode {
BAD_REQUEST
UNAUTHORIZED
}
union DiscoverFeedResult = DiscoverFeedError | DiscoverFeedSuccess
type DiscoverFeedSuccess {
feeds: [DiscoverFeed]!
}
type DiscoverTopic {
description: String!
name: String!
}
type EditDiscoverFeedError {
errorCodes: [EditDiscoverFeedErrorCode!]!
}
enum EditDiscoverFeedErrorCode {
BAD_REQUEST
NOT_FOUND
UNAUTHORIZED
}
input EditDiscoverFeedInput {
feedId: ID!
name: String!
}
union EditDiscoverFeedResult = EditDiscoverFeedError | EditDiscoverFeedSuccess
type EditDiscoverFeedSuccess {
id: ID!
}
type EmptyTrashError {
errorCodes: [EmptyTrashErrorCode!]!
}
@ -660,6 +794,21 @@ type EmptyTrashSuccess {
success: Boolean
}
type ExportToIntegrationError {
errorCodes: [ExportToIntegrationErrorCode!]!
}
enum ExportToIntegrationErrorCode {
FAILED_TO_CREATE_TASK
UNAUTHORIZED
}
union ExportToIntegrationResult = ExportToIntegrationError | ExportToIntegrationSuccess
type ExportToIntegrationSuccess {
task: Task!
}
type Feature {
createdAt: Date!
expiresAt: Date
@ -816,6 +965,37 @@ type GenerateApiKeySuccess {
apiKey: ApiKey!
}
type GetDiscoverFeedArticleError {
errorCodes: [GetDiscoverFeedArticleErrorCode!]!
}
enum GetDiscoverFeedArticleErrorCode {
BAD_REQUEST
NOT_FOUND
UNAUTHORIZED
}
union GetDiscoverFeedArticleResults = GetDiscoverFeedArticleError | GetDiscoverFeedArticleSuccess
type GetDiscoverFeedArticleSuccess {
discoverArticles: [DiscoverFeedArticle]
pageInfo: PageInfo!
}
type GetDiscoverTopicError {
errorCodes: [GetDiscoverTopicErrorCode!]!
}
enum GetDiscoverTopicErrorCode {
UNAUTHORIZED
}
union GetDiscoverTopicResults = GetDiscoverTopicError | GetDiscoverTopicSuccess
type GetDiscoverTopicSuccess {
discoverTopics: [DiscoverTopic!]
}
type GetFollowersError {
errorCodes: [GetFollowersErrorCode!]!
}
@ -968,12 +1148,27 @@ type Integration {
enabled: Boolean!
id: ID!
name: String!
settings: JSON
taskName: String
token: String!
type: IntegrationType!
updatedAt: Date
}
type IntegrationError {
errorCodes: [IntegrationErrorCode!]!
}
enum IntegrationErrorCode {
NOT_FOUND
}
union IntegrationResult = IntegrationError | IntegrationSuccess
type IntegrationSuccess {
integration: Integration!
}
enum IntegrationType {
EXPORT
IMPORT
@ -1221,6 +1416,7 @@ type MoveToFolderSuccess {
}
type Mutation {
addDiscoverFeed(input: AddDiscoverFeedInput!): AddDiscoverFeedResult!
addPopularRead(name: String!): AddPopularReadResult!
bulkAction(action: BulkActionType!, arguments: JSON, async: Boolean, expectedCount: Int, labelIds: [ID!], query: String!): BulkActionResult!
createArticle(input: CreateArticleInput!): CreateArticleResult!
@ -1230,6 +1426,8 @@ type Mutation {
createLabel(input: CreateLabelInput!): CreateLabelResult!
createNewsletterEmail(input: CreateNewsletterEmailInput): CreateNewsletterEmailResult!
deleteAccount(userID: ID!): DeleteAccountResult!
deleteDiscoverArticle(input: DeleteDiscoverArticleInput!): DeleteDiscoverArticleResult!
deleteDiscoverFeed(input: DeleteDiscoverFeedInput!): DeleteDiscoverFeedResult!
deleteFilter(id: ID!): DeleteFilterResult!
deleteHighlight(highlightId: ID!): DeleteHighlightResult!
deleteIntegration(id: ID!): DeleteIntegrationResult!
@ -1237,7 +1435,9 @@ type Mutation {
deleteNewsletterEmail(newsletterEmailId: ID!): DeleteNewsletterEmailResult!
deleteRule(id: ID!): DeleteRuleResult!
deleteWebhook(id: ID!): DeleteWebhookResult!
editDiscoverFeed(input: EditDiscoverFeedInput!): EditDiscoverFeedResult!
emptyTrash: EmptyTrashResult!
exportToIntegration(integrationId: ID!): ExportToIntegrationResult!
fetchContent(id: ID!): FetchContentResult!
generateApiKey(input: GenerateApiKeyInput!): GenerateApiKeyResult!
googleLogin(input: GoogleLoginInput!): LoginResult!
@ -1257,6 +1457,7 @@ type Mutation {
reportItem(input: ReportItemInput!): ReportItemResult!
revokeApiKey(id: ID!): RevokeApiKeyResult!
saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult!
saveDiscoverArticle(input: SaveDiscoverArticleInput!): SaveDiscoverArticleResult!
saveFile(input: SaveFileInput!): SaveResult!
saveFilter(input: SaveFilterInput!): SaveFilterResult!
savePage(input: SavePageInput!): SaveResult!
@ -1412,11 +1613,15 @@ type Query {
article(format: String, slug: String!, username: String!): ArticleResult!
articleSavingRequest(id: ID, url: String): ArticleSavingRequestResult!
deviceTokens: DeviceTokensResult!
discoverFeeds: DiscoverFeedResult!
discoverTopics: GetDiscoverTopicResults!
feeds(input: FeedsInput!): FeedsResult!
filters: FiltersResult!
getDiscoverFeedArticles(after: String, discoverTopicId: String!, feedId: ID, first: Int): GetDiscoverFeedArticleResults!
getUserPersonalization: GetUserPersonalizationResult!
groups: GroupsResult!
hello: String
integration(name: String!): IntegrationResult!
integrations: IntegrationsResult!
labels: LabelsResult!
me: User
@ -1651,6 +1856,7 @@ type Rule {
createdAt: Date!
enabled: Boolean!
eventTypes: [RuleEventType!]!
failedAt: Date
filter: String!
id: ID!
name: String!
@ -1719,6 +1925,29 @@ type SaveArticleReadingProgressSuccess {
updatedArticle: Article!
}
type SaveDiscoverArticleError {
errorCodes: [SaveDiscoverArticleErrorCode!]!
}
enum SaveDiscoverArticleErrorCode {
BAD_REQUEST
NOT_FOUND
UNAUTHORIZED
}
input SaveDiscoverArticleInput {
discoverArticleId: ID!
locale: String
timezone: String
}
union SaveDiscoverArticleResult = SaveDiscoverArticleError | SaveDiscoverArticleSuccess
type SaveDiscoverArticleSuccess {
saveId: String!
url: String!
}
type SaveError {
errorCodes: [SaveErrorCode!]!
message: String
@ -1832,6 +2061,7 @@ enum SearchErrorCode {
}
type SearchItem {
aiSummary: String
annotation: String
archivedAt: Date
author: String
@ -1840,6 +2070,7 @@ type SearchItem {
contentReader: ContentReader!
createdAt: Date!
description: String
directionality: DirectionalityType
feedContent: String
folder: String!
highlights: [Highlight!]
@ -2001,6 +2232,7 @@ input SetIntegrationInput {
id: ID
importItemState: ImportItemState
name: String!
settings: JSON
syncedAt: Date
taskName: String
token: String!
@ -2302,6 +2534,25 @@ type SyncUpdatedItemEdge {
updateReason: UpdateReason!
}
type Task {
cancellable: Boolean
createdAt: Date!
failedReason: String
id: ID!
name: String!
progress: Float
runningTime: Int
state: TaskState!
}
enum TaskState {
CANCELLED
FAILED
PENDING
RUNNING
SUCCEEDED
}
type TypeaheadSearchError {
errorCodes: [TypeaheadSearchErrorCode!]!
}
@ -2733,6 +2984,8 @@ enum UploadImportFileType {
type User {
email: String
featureList: [Feature!]
features: [String]
followersCount: Int
friendsCount: Int
id: ID!

93
packages/api/src/jobs/ai-summarize.ts Normal file
View file

@ -0,0 +1,93 @@
import { logger } from '../utils/logger'
import { loadSummarizationChain } from 'langchain/chains'
import { ChatOpenAI } from '@langchain/openai'
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'
import { authTrx } from '../repository'
import { libraryItemRepository } from '../repository/library_item'
import { htmlToMarkdown } from '../utils/parser'
import { AISummary } from '../entity/AISummary'
import { LibraryItemState } from '../entity/library_item'
import { getAISummary } from '../services/ai-summaries'
export interface AISummarizeJobData {
userId: string
promptId?: string
libraryItemId: string
}
export const AI_SUMMARIZE_JOB_NAME = 'ai-summary-job'
export const aiSummarize = async (jobData: AISummarizeJobData) => {
try {
const libraryItem = await authTrx(
async (tx) =>
tx
.withRepository(libraryItemRepository)
.findById(jobData.libraryItemId),
undefined,
jobData.userId
)
if (!libraryItem || libraryItem.state !== LibraryItemState.Succeeded) {
logger.info(
`Not ready to summarize library item job state: ${
libraryItem?.state ?? 'null'
}`
)
return
}
const existingSummary = await getAISummary({
userId: jobData.userId,
idx: 'latest',
libraryItemId: jobData.libraryItemId,
})
if (existingSummary) {
logger.info(
`Library item already has a summary: ${jobData.libraryItemId}`
)
return
}
const llm = new ChatOpenAI({
configuration: {
apiKey: process.env.OPENAI_API_KEY,
},
})
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize: 2000,
})
const document = htmlToMarkdown(libraryItem.readableContent)
const docs = await textSplitter.createDocuments([document])
const chain = loadSummarizationChain(llm, {
type: 'map_reduce', // you can choose from map_reduce, stuff or refine
verbose: true, // to view the steps in the console
})
const response = await chain.call({
input_documents: docs,
})
if (typeof response.text !== 'string') {
logger.error(`AI summary did not return text`)
return
}
const summary = response.text
const _ = await authTrx(
async (t) => {
return t.getRepository(AISummary).save({
user: { id: jobData.userId },
libraryItem: { id: jobData.libraryItemId },
title: libraryItem.title,
slug: libraryItem.slug,
summary: summary,
})
},
undefined,
jobData.userId
)
} catch (err) {
console.log('error creating summary: ', err)
}
}

2
packages/api/src/jobs/find_thumbnail.ts
View file

@ -59,7 +59,7 @@ const getImageSize = async (src: string): Promise<ImageSize | null> => {
height,
}
} catch (e) {
logger.error(e)
logger.error('get image size error', e)
return null
}
}

59
packages/api/src/jobs/integration/export_all_items.ts
View file

@ -1,8 +1,11 @@
import { IntegrationType } from '../../entity/integration'
import { findIntegration } from '../../services/integrations'
import {
findIntegration,
getIntegrationClient,
updateIntegration,
} from '../../services/integrations'
import { findRecentLibraryItems } from '../../services/library_item'
import { findActiveUser } from '../../services/user'
import { enqueueExportItem } from '../../utils/createTask'
import { logger } from '../../utils/logger'
export interface ExportAllItemsJobData {
@ -39,17 +42,23 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => {
return
}
const maxItems = 1000
const limit = 100
const client = getIntegrationClient(
integration.name,
integration.token,
integration
)
const maxItems = 100
const limit = 10
let offset = 0
// get max 1000 most recent items from the database
// get max 100 most recent items from the database
while (offset < maxItems) {
const libraryItems = await findRecentLibraryItems(userId, limit, offset)
if (libraryItems.length === 0) {
logger.info('no library items found', {
userId,
})
return
break
}
logger.info('enqueuing export item...', {
@ -58,18 +67,44 @@ export const exportAllItems = async (jobData: ExportAllItemsJobData) => {
integrationId,
})
await enqueueExportItem({
userId,
libraryItemIds: libraryItems.map((item) => item.id),
integrationId,
const synced = await client.export(libraryItems)
if (!synced) {
logger.error('failed to export item', jobData)
continue
}
const syncedAt = new Date()
logger.info('updating integration...', {
...jobData,
syncedAt,
})
// update integration syncedAt if successful
const updated = await updateIntegration(
integration.id,
{
syncedAt,
},
userId
)
logger.info('integration updated', {
...jobData,
updated,
})
offset += libraryItems.length
logger.info('exported items', {
userId,
...jobData,
offset,
integrationId,
})
}
logger.info('exported all items', {
...jobData,
offset,
})
// clear task name in integration
await updateIntegration(integration.id, { taskName: null }, userId)
}

35
packages/api/src/jobs/integration/export_item.ts
View file

@ -37,19 +37,23 @@ export const exportItem = async (jobData: ExportItemJobData) => {
await Promise.all(
integrations.map(async (integration) => {
const logObject = {
userId,
integrationId: integration.id,
}
logger.info('exporting item...', logObject)
try {
const client = getIntegrationClient(integration.name)
const logObject = {
userId,
integrationId: integration.id,
}
logger.info('exporting item...', logObject)
const synced = await client.export(integration.token, libraryItems)
const client = getIntegrationClient(
integration.name,
integration.token,
integration
)
const synced = await client.export(libraryItems)
if (!synced) {
logger.error('failed to export item', logObject)
return Promise.resolve(false)
return false
}
const syncedAt = new Date()
@ -70,12 +74,15 @@ export const exportItem = async (jobData: ExportItemJobData) => {
...logObject,
updated,
})
return Promise.resolve(true)
} catch (err) {
logger.error('export with integration failed', err)
return Promise.resolve(false)
} catch (error) {
logger.error('failed to export item', {
userId,
integrationId: integration.id,
error,
})
}
})
)
return true
}

511
packages/api/src/jobs/process-youtube-video.ts Normal file
View file

@ -0,0 +1,511 @@
import { Storage } from '@google-cloud/storage'
import { PromptTemplate } from '@langchain/core/prompts'
import { OpenAI } from '@langchain/openai'
import { parseHTML } from 'linkedom'
import showdown from 'showdown'
import * as stream from 'stream'
import { Chapter, Client as YouTubeClient } from 'youtubei'
import { LibraryItem, LibraryItemState } from '../entity/library_item'
import { env } from '../env'
import { authTrx } from '../repository'
import { libraryItemRepository } from '../repository/library_item'
import { FeatureName, findGrantedFeatureByName } from '../services/features'
import { enqueueProcessYouTubeTranscript } from '../utils/createTask'
import { stringToHash } from '../utils/helpers'
import { logger } from '../utils/logger'
import { parsePreparedContent } from '../utils/parser'
import { videoIdFromYouTubeUrl } from '../utils/youtube'
export interface ProcessYouTubeVideoJobData {
userId: string
libraryItemId: string
}
export const PROCESS_YOUTUBE_VIDEO_JOB_NAME = 'process-youtube-video'
export const PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME = 'process-youtube-transcript'
const TRANSCRIPT_PLACEHOLDER_TEXT =
'* Omnivore is preparing a transcript for this video'
const calculateWordCount = (durationInSeconds: number): number => {
// Calculate word count using the formula: word count = read time (in seconds) * words per second
// Assuming average reading speed is 235 words per minute (or about 3.92 words per second)
const wordsPerSecond = 3.92
const wordCount = Math.round(durationInSeconds * wordsPerSecond)
return wordCount
}
interface ChapterProperties {
title: string
start: number
}
interface TranscriptProperties {
text: string
start: number
duration: number
}
export const addTranscriptChapters = (
chapters: ChapterProperties[],
transcript: TranscriptProperties[]
): TranscriptProperties[] => {
chapters.sort((a, b) => a.start - b.start)
for (const chapter of chapters) {
const startOffset = chapter.start
const title = '\n\n## ' + chapter.title + '\n\n'
const index = transcript.findIndex(
(textItem) => textItem.start > startOffset
)
if (index !== -1) {
transcript.splice(index, 0, {
text: title,
duration: 1,
start: startOffset,
})
} else {
transcript.push({ text: title, duration: 0, start: startOffset })
}
}
return transcript
}
const createTranscriptHash = (transcript: TranscriptProperties[]): string => {
const rawTranscript = transcript.map((item) => item.text).join(' ')
return stringToHash(rawTranscript)
}
export const createTranscriptHTML = async (
videoId: string,
transcript: TranscriptProperties[]
): Promise<string> => {
let transcriptMarkdown = ''
const transcriptHash = createTranscriptHash(transcript)
const promptHash = stringToHash(process.env.YOUTUBE_TRANSCRIPT_PROMPT ?? '')
if (process.env.YOUTUBE_TRANSCRIPT_PROMPT && process.env.OPENAI_API_KEY) {
const cachedTranscriptHTML = await fetchCachedYouTubeTranscript(
videoId,
transcriptHash,
promptHash
)
if (cachedTranscriptHTML) {
return cachedTranscriptHTML
}
const llm = new OpenAI({
modelName: 'gpt-4-0125-preview',
configuration: {
apiKey: process.env.OPENAI_API_KEY,
},
})
const promptTemplate = PromptTemplate.fromTemplate(
`${process.env.YOUTUBE_TRANSCRIPT_PROMPT}
{transcriptData}`
)
const chain = promptTemplate.pipe(llm)
const result = await chain.invoke({
transcriptData: transcript.map((item) => item.text).join(' '),
})
transcriptMarkdown = result
}
// If the LLM didn't give us enough data fallback to the raw template
if (transcriptMarkdown.length < 1) {
transcriptMarkdown = transcript.map((item) => item.text).join(' ')
}
const converter = new showdown.Converter({
backslashEscapesHTMLTags: true,
})
const transcriptHTML = converter.makeHtml(transcriptMarkdown)
if (process.env.YOUTUBE_TRANSCRIPT_PROMPT && process.env.OPENAI_API_KEY) {
await cacheYouTubeTranscript(
videoId,
transcriptHash,
promptHash,
transcriptHTML
)
}
return transcriptHTML
}
export const addTranscriptToReadableContent = async (
originalUrl: string,
originalHTML: string,
transcriptHTML: string
): Promise<string | undefined> => {
const html = parseHTML(originalHTML)
const transcriptNode = html.document.querySelector(
'#_omnivore_youtube_transcript'
)
if (transcriptNode) {
transcriptNode.innerHTML = transcriptHTML
} else {
const div = html.document.createElement('div')
div.innerHTML = transcriptHTML
html.document.body.appendChild(div)
}
const preparedDocument = {
document: html.document.toString(),
pageInfo: {},
}
const updatedContent = await parsePreparedContent(
originalUrl,
preparedDocument,
true
)
return updatedContent.parsedContent?.content
}
export const addTranscriptPlaceholdReadableContent = async (
originalUrl: string,
originalHTML: string
): Promise<string | undefined> => {
const html = parseHTML(originalHTML)
const transcriptNode = html.document.querySelector(
'#_omnivore_youtube_transcript'
)
if (transcriptNode) {
transcriptNode.innerHTML = TRANSCRIPT_PLACEHOLDER_TEXT
} else {
const div = html.document.createElement('div')
div.innerHTML = TRANSCRIPT_PLACEHOLDER_TEXT
html.document.body.appendChild(div)
}
const preparedDocument = {
document: html.document.toString(),
pageInfo: {},
}
const updatedContent = await parsePreparedContent(
originalUrl,
preparedDocument,
true
)
return updatedContent.parsedContent?.content
}
async function readStringFromStorage(
bucketName: string,
fileName: string
): Promise<string | undefined> {
try {
const storage = env.fileUpload?.gcsUploadSAKeyFilePath
? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
: new Storage()
const existsResponse = await storage
.bucket(bucketName)
.file(fileName)
.exists()
const exists = existsResponse[0]
if (!exists) {
throw new Error(
`File '${fileName}' does not exist in bucket '${bucketName}'.`
)
}
// Download the file contents as a string
const fileContentResponse = await storage
.bucket(bucketName)
.file(fileName)
.download()
const fileContent = fileContentResponse[0].toString()
return fileContent
} catch (error) {
// This isn't a catastrophic error it just means the file doesn't exist
logger.info('Error downloading file:', error)
return undefined
}
}
const writeStringToStorage = async (
bucketName: string,
fileName: string,
content: string
): Promise<void> => {
try {
const storage = env.fileUpload?.gcsUploadSAKeyFilePath
? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
: new Storage()
const writableStream = storage
.bucket(bucketName)
.file(fileName)
.createWriteStream()
// Convert the string content to a readable stream
const readableStream = new stream.Readable()
readableStream.push(content)
readableStream.push(null) // Signal the end of the stream
// Pipe the readable stream to the writable stream to upload the file content
await new Promise((resolve, reject) => {
readableStream
.pipe(writableStream)
.on('finish', resolve)
.on('error', reject)
})
logger.info(
`File '${fileName}' uploaded successfully to bucket '${bucketName}'.`
)
} catch (error) {
logger.error('Error uploading file:', error)
throw error
}
}
const fetchCachedYouTubeTranscript = async (
videoId: string,
transcriptHash: string,
promptHash: string
): Promise<string | undefined> => {
const bucketName = env.fileUpload.gcsUploadBucket
try {
return await readStringFromStorage(
bucketName,
`youtube-transcripts/${videoId}/${transcriptHash}.${promptHash}.html`
)
} catch (err) {
logger.info(`unable to fetch cached transcript`, { error: err })
}
return undefined
}
const cacheYouTubeTranscript = async (
videoId: string,
transcriptHash: string,
promptHash: string,
transcript: string
): Promise<void> => {
const bucketName = env.fileUpload.gcsUploadBucket
try {
await writeStringToStorage(
bucketName,
`youtube-transcripts/${videoId}/${transcriptHash}.${promptHash}.html`,
transcript
)
} catch (err) {
logger.info(`unable to cache transcript`, { error: err })
}
}
export const processYouTubeVideo = async (
jobData: ProcessYouTubeVideoJobData
) => {
let videoURL: URL | undefined
try {
const libraryItem = await authTrx(
async (tx) =>
tx
.withRepository(libraryItemRepository)
.findById(jobData.libraryItemId),
undefined,
jobData.userId
)
if (
!libraryItem ||
libraryItem.state !== LibraryItemState.Succeeded ||
!libraryItem.originalContent
) {
logger.info(
`Not ready to get YouTube metadata job state: ${
libraryItem?.state ?? 'null'
}`
)
return
}
videoURL = new URL(libraryItem.originalUrl)
const videoId = videoIdFromYouTubeUrl(libraryItem.originalUrl)
if (!videoId) {
logger.warning('no video id for supplied youtube url', {
url: libraryItem.originalUrl,
})
return
}
let needsUpdate = false
const youtube = new YouTubeClient()
const video = await youtube.getVideo(videoId)
if (!video) {
logger.warning('no video found for youtube url', {
url: libraryItem.originalUrl,
})
return
}
if (video.description && libraryItem.description !== video.description) {
needsUpdate = true
libraryItem.description = video.description
}
let duration = -1
if ('duration' in video && video.duration > 0) {
needsUpdate = true
libraryItem.wordCount = calculateWordCount(video.duration)
duration = video.duration
}
if (video.uploadDate && !Number.isNaN(Date.parse(video.uploadDate))) {
needsUpdate = true
libraryItem.publishedAt = new Date(video.uploadDate)
}
if (
await findGrantedFeatureByName(
FeatureName.YouTubeTranscripts,
jobData.userId
)
) {
if ('getTranscript' in video && duration > 0 && duration < 1801) {
// If the video has a transcript available, put a placehold in and
// enqueue a job to process the full transcript
const updatedContent = await addTranscriptPlaceholdReadableContent(
libraryItem.originalUrl,
libraryItem.originalContent
)
if (updatedContent) {
needsUpdate = true
libraryItem.readableContent = updatedContent
}
await enqueueProcessYouTubeTranscript({
videoId,
...jobData,
})
}
}
if (needsUpdate) {
const updated = await authTrx(
async (t) => {
return t
.getRepository(LibraryItem)
.update(jobData.libraryItemId, libraryItem)
},
undefined,
jobData.userId
)
if (!updated) {
logger.warning('could not updated library item')
}
}
} catch (err) {
logger.warning('error getting youtube metadata: ', {
err,
jobData,
videoURL,
})
}
}
export interface ProcessYouTubeTranscriptJobData {
userId: string
videoId: string
libraryItemId: string
}
export const processYouTubeTranscript = async (
jobData: ProcessYouTubeTranscriptJobData
) => {
try {
const libraryItem = await authTrx(
async (tx) =>
tx
.withRepository(libraryItemRepository)
.findById(jobData.libraryItemId),
undefined,
jobData.userId
)
if (
!libraryItem ||
libraryItem.state !== LibraryItemState.Succeeded ||
!libraryItem.originalContent
) {
logger.info(
`Not ready to get YouTube metadata job state: ${
libraryItem?.state ?? 'null'
}`
)
return
}
let needsUpdate = false
const youtube = new YouTubeClient()
const video = await youtube.getVideo(jobData.videoId)
if (!video) {
logger.warning('no video found for youtube url', {
url: libraryItem.originalUrl,
})
return
}
let chapters: Chapter[] = []
if ('chapters' in video) {
chapters = video.chapters
}
let transcript: TranscriptProperties[] | undefined = undefined
if ('getTranscript' in video) {
transcript = await video.getTranscript()
}
if (transcript) {
if (chapters) {
transcript = addTranscriptChapters(chapters, transcript)
}
const transcriptHTML = await createTranscriptHTML(
jobData.videoId,
transcript
)
const updatedContent = await addTranscriptToReadableContent(
libraryItem.originalUrl,
libraryItem.originalContent,
transcriptHTML
)
if (updatedContent) {
needsUpdate = true
libraryItem.readableContent = updatedContent
}
}
if (needsUpdate) {
const updated = await authTrx(
async (t) => {
return t
.getRepository(LibraryItem)
.update(jobData.libraryItemId, libraryItem)
},
undefined,
jobData.userId
)
if (!updated) {
logger.warning('could not updated library item')
}
}
} catch (err) {
logger.warning('error getting youtube transcript: ', { err, jobData })
}
}

9
packages/api/src/jobs/rss/refreshAllFeeds.ts
View file

@ -43,7 +43,7 @@ export const refreshAllFeeds = async (db: DataSource): Promise<boolean> => {
AND (s.scheduled_at <= NOW() OR s.scheduled_at IS NULL)
AND u.status = $4
GROUP BY
s.url
url
`,
['RSS', 'ACTIVE', 'following', 'ACTIVE']
)) as RssSubscriptionGroup[]
@ -76,7 +76,10 @@ const updateSubscriptionGroup = async (
refreshContext: RSSRefreshContext
) => {
let feedURL = group.url
const userList = JSON.stringify(group.userIds.sort())
const userIds = group.userIds
// sort the user ids so that the job id is consistent
// [...userIds] creates a shallow copy, so sort() does not mutate the original
const userList = JSON.stringify([...userIds].sort())
if (!feedURL) {
logger.error('no url for feed group', group)
return
@ -105,7 +108,7 @@ const updateSubscriptionGroup = async (
scheduledTimestamps: group.scheduledDates.map((timestamp) =>
timestamp.getTime()
), // unix timestamp in milliseconds
userIds: group.userIds,
userIds,
fetchContentTypes: group.fetchContentTypes,
folders: group.folders,
}

15
packages/api/src/jobs/rss/refreshFeed.ts
View file

@ -48,12 +48,14 @@ export const isRefreshFeedRequest = (data: any): data is RefreshFeedRequest => {
// link can be a string or an object
type RssFeedItemLink = string | { $: { rel?: string; href: string } }
type RssFeedItemAuthor = string | { name: string }
type RssFeed = Parser.Output<{
published?: string
updated?: string
created?: string
link?: RssFeedItemLink
links?: RssFeedItemLink[]
author?: RssFeedItemAuthor
}> & {
lastBuildDate?: string
'syn:updatePeriod'?: string
@ -359,6 +361,7 @@ const createItemWithFeedContent = async (
clientRequestId: '',
author: item.creator,
previewImage,
labels: [{ name: 'RSS' }],
},
user
)
@ -386,6 +389,7 @@ const parser = new Parser({
'created',
['media:content', 'media:content', { keepArray: true }],
['media:thumbnail'],
'author',
],
feed: [
'lastBuildDate',
@ -473,6 +477,14 @@ const getLink = (
return url
}
// get author
const getAuthor = (author: RssFeedItemAuthor) => {
if (typeof author === 'string') {
return author
}
return author.name
}
const processSubscription = async (
fetchContentTasks: Map<string, FetchContentTask>,
subscriptionId: string,
@ -536,10 +548,13 @@ const processSubscription = async (
throw new Error('Invalid feed item link')
}
const creator = item.creator || (item.author && getAuthor(item.author))
const feedItem = {
...item,
isoDate,
link,
creator,
}
const publishedAt = feedItem.isoDate

92
packages/api/src/jobs/trigger_rule.ts
View file

@ -1,27 +1,33 @@
import { LiqeQuery } from '@omnivore/liqe'
import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source'
import { LibraryItem, LibraryItemState } from '../entity/library_item'
import { Rule, RuleAction, RuleActionType, RuleEventType } from '../entity/rule'
import { addLabelsToLibraryItem } from '../services/labels'
import {
SearchArgs,
filterItemEvents,
ItemEvent,
RequiresSearchQueryError,
searchLibraryItems,
softDeleteLibraryItem,
updateLibraryItem,
} from '../services/library_item'
import { findEnabledRules } from '../services/rules'
import { findEnabledRules, markRuleAsFailed } from '../services/rules'
import { sendPushNotifications } from '../services/user'
import { logger } from '../utils/logger'
import { parseSearchQuery } from '../utils/search'
export interface TriggerRuleJobData {
libraryItemId: string
userId: string
ruleEventType: RuleEventType
data: ItemEvent
}
interface RuleActionObj {
libraryItemId: string
userId: string
action: RuleAction
libraryItem: LibraryItem
data: ItemEvent | LibraryItem
}
type RuleActionFunc = (obj: RuleActionObj) => Promise<unknown>
@ -33,19 +39,19 @@ const addLabels = async (obj: RuleActionObj) => {
return addLabelsToLibraryItem(
labelIds,
obj.libraryItem.id,
obj.libraryItemId,
obj.userId,
'system'
)
}
const deleteLibraryItem = async (obj: RuleActionObj) => {
return softDeleteLibraryItem(obj.libraryItem.id, obj.userId)
return softDeleteLibraryItem(obj.libraryItemId, obj.userId)
}
const archivePage = async (obj: RuleActionObj) => {
return updateLibraryItem(
obj.libraryItem.id,
obj.libraryItemId,
{ archivedAt: new Date(), state: LibraryItemState.Archived },
obj.userId,
undefined,
@ -56,7 +62,7 @@ const archivePage = async (obj: RuleActionObj) => {
const markPageAsRead = async (obj: RuleActionObj) => {
return readingProgressDataSource.updateReadingProgress(
obj.userId,
obj.libraryItem.id,
obj.libraryItemId,
{
readingProgressPercent: 100,
readingProgressTopPercent: 100,
@ -66,13 +72,18 @@ const markPageAsRead = async (obj: RuleActionObj) => {
}
const sendNotification = async (obj: RuleActionObj) => {
const item = obj.libraryItem
const item = obj.data
const message = {
title: item.author || item.siteName || 'Omnivore',
body: item.title,
title: item.author?.toString() || item.siteName?.toString() || 'Omnivore',
body: item.title?.toString(),
image: item.thumbnail,
}
const data = {
folder: item.folder?.toString() || 'inbox',
libraryItemId: obj.libraryItemId,
}
return sendPushNotifications(obj.userId, message, 'rule')
return sendPushNotifications(obj.userId, message, 'rule', data)
}
const getRuleAction = (actionType: RuleActionType): RuleActionFunc => {
@ -91,36 +102,59 @@ const getRuleAction = (actionType: RuleActionType): RuleActionFunc => {
}
const triggerActions = async (
libraryItemId: string,
userId: string,
rules: Rule[],
data: TriggerRuleJobData
data: ItemEvent
) => {
const actionPromises: Promise<unknown>[] = []
for (const rule of rules) {
const itemId = data.libraryItemId
const searchArgs: SearchArgs = {
includeContent: false,
includeDeleted: false,
includePending: false,
size: 1,
query: `(${rule.filter}) AND includes:${itemId}`,
}
let ast: LiqeQuery
let results: (ItemEvent | LibraryItem)[]
try {
ast = parseSearchQuery(rule.filter)
} catch (error) {
logger.error('Error parsing filter in rules', error)
await markRuleAsFailed(rule.id, userId)
const libraryItems = await searchLibraryItems(searchArgs, userId)
if (libraryItems.count === 0) {
logger.info(`No pages found for rule ${rule.id}`)
continue
}
const libraryItem = libraryItems.libraryItems[0]
// filter library item by metadata
try {
results = filterItemEvents(ast, [data])
} catch (error) {
if (error instanceof RequiresSearchQueryError) {
logger.info('Failed to filter items by metadata, running search query')
const searchResult = await searchLibraryItems(
{
query: `includes:${libraryItemId} AND (${rule.filter})`,
size: 1,
},
userId
)
results = searchResult.libraryItems
} else {
logger.error('Error filtering item events', error)
await markRuleAsFailed(rule.id, userId)
continue
}
}
if (results.length === 0) {
logger.info(`No items found for rule ${rule.id}`)
continue
}
for (const action of rule.actions) {
const actionFunc = getRuleAction(action.type)
const actionObj: RuleActionObj = {
libraryItemId,
userId,
action,
libraryItem,
data: results[0],
}
actionPromises.push(actionFunc(actionObj))
@ -130,12 +164,12 @@ const triggerActions = async (
try {
await Promise.all(actionPromises)
} catch (error) {
logger.error(error)
logger.error('Error triggering rule actions', error)
}
}
export const triggerRule = async (data: TriggerRuleJobData) => {
const { userId, ruleEventType } = data
export const triggerRule = async (jobData: TriggerRuleJobData) => {
const { userId, ruleEventType, data, libraryItemId } = jobData
// get rules by calling api
const rules = await findEnabledRules(userId, ruleEventType)
@ -144,7 +178,7 @@ export const triggerRule = async (data: TriggerRuleJobData) => {
return false
}
await triggerActions(userId, rules, data)
await triggerActions(libraryItemId, userId, rules, data)
return true
}

92
packages/api/src/pubsub.ts
View file

@ -3,24 +3,20 @@ import express from 'express'
import { RuleEventType } from './entity/rule'
import { env } from './env'
import { ReportType } from './generated/graphql'
import { Merge } from './util'
import {
enqueueExportItem,
enqueueProcessYouTubeVideo,
enqueueTriggerRuleJob,
enqueueWebhookJob,
} from './utils/createTask'
import { deepDelete } from './utils/helpers'
import { buildLogger } from './utils/logger'
import { isYouTubeVideoURL } from './utils/youtube'
const logger = buildLogger('pubsub')
const client = new PubSub()
type EntityData<T> = Merge<T, { libraryItemId: string }>
export const createPubSubClient = (): PubsubClient => {
const fieldsToDelete = ['user'] as const
const publish = (topicName: string, msg: Buffer): Promise<void> => {
if (env.dev.isLocal) {
logger.info(`Publishing ${topicName}: ${msg.toString()}`)
@ -50,18 +46,19 @@ export const createPubSubClient = (): PubsubClient => {
Buffer.from(JSON.stringify({ userId, email, name, username }))
)
},
entityCreated: async <T>(
entityCreated: async <T extends Record<string, any>>(
type: EntityType,
data: EntityData<T>,
userId: string
data: T,
userId: string,
libraryItemId: string
): Promise<void> => {
const libraryItemId = data.libraryItemId
// queue trigger rule job
if (type === EntityType.PAGE) {
await enqueueTriggerRuleJob({
userId,
ruleEventType: RuleEventType.PageCreated,
libraryItemId,
data,
})
}
// queue export item job
@ -70,11 +67,6 @@ export const createPubSubClient = (): PubsubClient => {
libraryItemIds: [libraryItemId],
})
const cleanData = deepDelete(
data as EntityData<T> & Record<typeof fieldsToDelete[number], unknown>,
[...fieldsToDelete]
)
await enqueueWebhookJob({
userId,
type,
@ -82,24 +74,39 @@ export const createPubSubClient = (): PubsubClient => {
data,
})
return publish(
'entityCreated',
Buffer.from(JSON.stringify({ type, userId, ...cleanData }))
)
},
entityUpdated: async <T>(
type: EntityType,
data: EntityData<T>,
userId: string
): Promise<void> => {
const libraryItemId = data.libraryItemId
if (type === EntityType.PAGE) {
// if (await findGrantedFeatureByName(FeatureName.AISummaries, userId)) {
// await enqueueAISummarizeJob({
// userId,
// libraryItemId,
// })
// }
const isItemWithURL = (data: any): data is { originalUrl: string } => {
return 'originalUrl' in data
}
if (isItemWithURL(data) && isYouTubeVideoURL(data['originalUrl'])) {
await enqueueProcessYouTubeVideo({
userId,
libraryItemId,
})
}
}
},
entityUpdated: async <T extends Record<string, any>>(
type: EntityType,
data: T,
userId: string,
libraryItemId: string
): Promise<void> => {
// queue trigger rule job
if (type === EntityType.PAGE) {
await enqueueTriggerRuleJob({
userId,
ruleEventType: RuleEventType.PageUpdated,
libraryItemId,
data,
})
}
// queue export item job
@ -108,32 +115,20 @@ export const createPubSubClient = (): PubsubClient => {
libraryItemIds: [libraryItemId],
})
const cleanData = deepDelete(
data as EntityData<T> & Record<typeof fieldsToDelete[number], unknown>,
[...fieldsToDelete]
)
await enqueueWebhookJob({
userId,
type,
action: 'updated',
data,
})
return publish(
'entityUpdated',
Buffer.from(JSON.stringify({ type, userId, ...cleanData }))
)
},
entityDeleted: (
entityDeleted: async (
type: EntityType,
id: string,
userId: string
): Promise<void> => {
return publish(
'entityDeleted',
Buffer.from(JSON.stringify({ type, id, userId }))
)
logger.info(`entityDeleted: ${type} ${id} ${userId}`)
await Promise.resolve()
},
reportSubmitted: (
submitterId: string,
@ -155,6 +150,7 @@ export enum EntityType {
PAGE = 'page',
HIGHLIGHT = 'highlight',
LABEL = 'label',
RSS_FEED = 'feed',
}
export interface PubsubClient {
@ -164,15 +160,17 @@ export interface PubsubClient {
name: string,
username: string
) => Promise<void>
entityCreated: <T>(
entityCreated: <T extends Record<string, any>>(
type: EntityType,
data: EntityData<T>,
userId: string
data: T,
userId: string,
libraryItemId: string
) => Promise<void>
entityUpdated: <T>(
entityUpdated: <T extends Record<string, any>>(
type: EntityType,
data: EntityData<T>,
userId: string
data: T,
userId: string,
libraryItemId: string
) => Promise<void>
entityDeleted: (type: EntityType, id: string, userId: string) => Promise<void>
reportSubmitted(

67
packages/api/src/queue-processor.ts
View file

@ -5,6 +5,7 @@
import {
ConnectionOptions,
Job,
JobState,
JobType,
Queue,
QueueEvents,
@ -13,6 +14,8 @@ import {
import express, { Express } from 'express'
import { appDataSource } from './data_source'
import { env } from './env'
import { TaskState } from './generated/graphql'
import { aiSummarize, AI_SUMMARIZE_JOB_NAME } from './jobs/ai-summarize'
import { bulkAction, BULK_ACTION_JOB_NAME } from './jobs/bulk_action'
import { callWebhook, CALL_WEBHOOK_JOB_NAME } from './jobs/call_webhook'
import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail'
@ -24,6 +27,12 @@ import {
exportItem,
EXPORT_ITEM_JOB_NAME,
} from './jobs/integration/export_item'
import {
processYouTubeTranscript,
processYouTubeVideo,
PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME,
PROCESS_YOUTUBE_VIDEO_JOB_NAME,
} from './jobs/process-youtube-video'
import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
import { refreshFeed } from './jobs/rss/refreshFeed'
import { savePageJob } from './jobs/save_page'
@ -75,6 +84,33 @@ export const getBackendQueue = async (): Promise<Queue | undefined> => {
return backendQueue
}
export const getJob = async (jobId: string) => {
const queue = await getBackendQueue()
if (!queue) {
return
}
return queue.getJob(jobId)
}
export const jobStateToTaskState = (
jobState: JobState | 'unknown'
): TaskState => {
switch (jobState) {
case 'completed':
return TaskState.Succeeded
case 'failed':
return TaskState.Failed
case 'active':
return TaskState.Running
case 'delayed':
return TaskState.Pending
case 'waiting':
return TaskState.Pending
default:
return TaskState.Pending
}
}
export const createWorker = (connection: ConnectionOptions) =>
new Worker(
QUEUE_NAME,
@ -113,8 +149,16 @@ export const createWorker = (connection: ConnectionOptions) =>
return callWebhook(job.data)
case EXPORT_ITEM_JOB_NAME:
return exportItem(job.data)
case AI_SUMMARIZE_JOB_NAME:
return aiSummarize(job.data)
case PROCESS_YOUTUBE_VIDEO_JOB_NAME:
return processYouTubeVideo(job.data)
case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
return processYouTubeTranscript(job.data)
case EXPORT_ALL_ITEMS_JOB_NAME:
return exportAllItems(job.data)
default:
logger.warning(`[queue-processor] unhandled job: ${job.name}`)
}
},
{
@ -171,7 +215,6 @@ const main = async () => {
let output = ''
const metrics: JobType[] = ['active', 'failed', 'completed', 'prioritized']
const counts = await queue.getJobCounts(...metrics)
console.log('counts: ', counts)
metrics.forEach((metric, idx) => {
output += `# TYPE omnivore_queue_messages_${metric} gauge\n`
@ -198,6 +241,18 @@ const main = async () => {
}
}
// Export the age of the oldest prioritized job in the queue
const oldestJobs = await queue.getJobs(['prioritized'], 0, 1, true)
if (oldestJobs.length > 0) {
const currentTime = Date.now()
const ageInSeconds = (currentTime - oldestJobs[0].timestamp) / 1000
output += `# TYPE omnivore_queue_messages_oldest_job_age_seconds gauge\n`
output += `omnivore_queue_messages_oldest_job_age_seconds{queue="${QUEUE_NAME}"} ${ageInSeconds}\n`
} else {
output += `# TYPE omnivore_queue_messages_oldest_job_age_seconds gauge\n`
output += `omnivore_queue_messages_oldest_job_age_seconds{queue="${QUEUE_NAME}"} ${0}\n`
}
res.status(200).setHeader('Content-Type', 'text/plain').send(output)
})
@ -253,6 +308,16 @@ const main = async () => {
process.on('SIGINT', () => gracefulShutdown('SIGINT'))
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'))
process.on('uncaughtException', function (err) {
// Handle the error safely
logger.error('Uncaught exception', err)
})
process.on('unhandledRejection', (reason, promise) => {
// Handle the error safely
logger.error('Unhandled Rejection at: Promise', { promise, reason })
})
}
// only call main if the file was called from the CLI and wasn't required from another module

19
packages/api/src/repository/index.ts
View file

@ -1,23 +1,28 @@
import * as httpContext from 'express-http-context2'
import { DatabaseError } from 'pg'
import {
EntityManager,
EntityTarget,
ObjectLiteral,
QueryBuilder,
QueryFailedError,
Repository,
} from 'typeorm'
import { DatabaseError } from 'pg'
import { appDataSource } from '../data_source'
import { Claims } from '../resolvers/types'
import { SetClaimsRole } from '../utils/dictionary'
export const getColumns = <T>(repository: Repository<T>): (keyof T)[] => {
export const getColumns = <T extends ObjectLiteral>(
repository: Repository<T>
): (keyof T)[] => {
return repository.metadata.columns.map(
(col) => col.propertyName
) as (keyof T)[]
}
export const getColumnsDbName = <T>(repository: Repository<T>): string[] => {
export const getColumnsDbName = <T extends ObjectLiteral>(
repository: Repository<T>
): string[] => {
return repository.metadata.columns.map((col) => col.databaseName)
}
@ -53,11 +58,15 @@ export const authTrx = async <T>(
})
}
export const getRepository = <T>(entity: EntityTarget<T>) => {
export const getRepository = <T extends ObjectLiteral>(
entity: EntityTarget<T>
) => {
return appDataSource.getRepository(entity)
}
export const queryBuilderToRawSql = <T>(q: QueryBuilder<T>): string => {
export const queryBuilderToRawSql = <T extends ObjectLiteral>(
q: QueryBuilder<T>
): string => {
const queryAndParams = q.getQueryAndParameters()
let sql = queryAndParams[0]
const params = queryAndParams[1]

10
packages/api/src/repository/library_item.ts
View file

@ -58,6 +58,8 @@ export const libraryItemRepository = appDataSource
},
createByPopularRead(name: string, userId: string) {
// set read_at to now and reading_progress_bottom_percent to 2
// so the items show up in continue reading section
return this.query(
`
INSERT INTO omnivore.library_item (
@ -73,7 +75,9 @@ export const libraryItemRepository = appDataSource
published_at,
site_name,
user_id,
word_count
word_count,
read_at,
reading_progress_bottom_percent
)
SELECT
slug,
@ -88,7 +92,9 @@ export const libraryItemRepository = appDataSource
published_at,
site_name,
$2,
word_count
word_count,
NOW(),
2
FROM
omnivore.popular_read
WHERE

190
packages/api/src/resolvers/discover_feeds/add.ts Normal file
View file

@ -0,0 +1,190 @@
/* eslint-disable @typescript-eslint/no-unused-vars */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/require-await */
import axios from 'axios'
import { XMLParser } from 'fast-xml-parser'
import { v4 } from 'uuid'
import { appDataSource } from '../../data_source'
import {
AddDiscoverFeedError,
AddDiscoverFeedErrorCode,
AddDiscoverFeedSuccess,
DiscoverFeed,
MutationAddDiscoverFeedArgs,
} from '../../generated/graphql'
import { authorized } from '../../utils/gql-utils'
import { RSS_PARSER_CONFIG } from '../../utils/parser'
const parser = new XMLParser({
ignoreAttributes: false,
parseTagValue: true,
ignoreDeclaration: false,
ignorePiTags: false,
})
type DiscoverFeedRows = {
rows: DiscoverFeed[]
}
const extractAtomData = (
url: string,
feed: {
title: string
subtitle?: string
icon?: string
}
): Partial<DiscoverFeed> => ({
description: feed.subtitle ?? '',
title: feed.title ?? url,
image: feed.icon,
link: url,
type: 'atom',
})
const extractRssData = (
url: string,
parsedXml: {
channel: {
title: string
description?: string
['sy:updateFrequency']: number
}
image: { url: string }
}
): Partial<DiscoverFeed> => ({
description: parsedXml.channel?.description ?? '',
title: parsedXml.channel.title ?? url,
image: parsedXml.image?.url,
link: url,
type: 'rss',
})
const handleExistingSubscription = async (
feed: DiscoverFeed,
userId: string
): Promise<AddDiscoverFeedSuccess | AddDiscoverFeedError> => {
// Add to existing, otherwise conflict.
const existingSubscription = await appDataSource.query(
'SELECT * FROM omnivore.discover_feed_subscription WHERE user_id = $1 and feed_id = $2',
[userId, feed.id]
)
if (existingSubscription.rows > 1) {
return {
__typename: 'AddDiscoverFeedError',
errorCodes: [AddDiscoverFeedErrorCode.Conflict],
}
}
await appDataSource.query(
'INSERT INTO omnivore.discover_feed_subscription(feed_id, user_id) VALUES($1, $2)',
[feed.id, userId]
)
return {
__typename: 'AddDiscoverFeedSuccess',
feed,
}
}
const addNewSubscription = async (
url: string,
userId: string
): Promise<AddDiscoverFeedSuccess | AddDiscoverFeedError> => {
// First things first, we need to validate that this is an actual RSS or ATOM feed.
const response = await axios.get(url, RSS_PARSER_CONFIG)
const content = response.data
const contentType = response.headers['content-type']
const isXML =
contentType?.includes('text/rss+xml') ||
contentType?.includes('text/atom+xml') ||
contentType?.includes('application/xml')
if (!isXML) {
return {
__typename: 'AddDiscoverFeedError',
errorCodes: [AddDiscoverFeedErrorCode.BadRequest],
}
}
const parsedFeed = parser.parse(content)
if (!parsedFeed?.rss && !parsedFeed['rdf:RDF'] && !parsedFeed['feed']) {
return {
__typename: 'AddDiscoverFeedError',
errorCodes: [AddDiscoverFeedErrorCode.BadRequest],
}
}
const feed =
parsedFeed?.rss || parsedFeed['rdf:RDF']
? extractRssData(url, parsedFeed.rss || parsedFeed['rdf:RDF'])
: extractAtomData(url, parsedFeed.feed)
if (!feed.title) {
return {
__typename: 'AddDiscoverFeedError',
errorCodes: [AddDiscoverFeedErrorCode.BadRequest],
}
}
const discoverFeedId = v4()
await appDataSource.query(
'INSERT INTO omnivore.discover_feed(id, title, link, image, type, description) VALUES($1, $2, $3, $4, $5, $6)',
[
discoverFeedId,
feed.title,
feed.link,
feed.image,
feed.type,
feed.description,
]
)
await appDataSource.query(
'INSERT INTO omnivore.discover_feed_subscription(feed_id, user_id) VALUES($2, $1)',
[userId, discoverFeedId]
)
return {
__typename: 'AddDiscoverFeedSuccess',
feed: { ...feed, id: discoverFeedId } as DiscoverFeed,
}
}
export const addDiscoverFeedResolver = authorized<
AddDiscoverFeedSuccess,
AddDiscoverFeedError,
MutationAddDiscoverFeedArgs
>(async (_, { input: { url } }, { uid, log, pubsub }) => {
try {
const existingFeed = (await appDataSource.query(
'SELECT id from omnivore.discover_feed where link = $1',
[url]
)) as DiscoverFeedRows
if (existingFeed.rows.length > 0) {
return await handleExistingSubscription(existingFeed.rows[0], uid)
}
const result = await addNewSubscription(url, uid)
// TODO: Add pubsub for new feed
// if (result.__typename == 'AddDiscoverFeedSuccess') {
// await pubsub.entityCreated(
// EntityType.RSS_FEED,
// { feed: result.feed, libraryItemId: 'NA' },
// uid
// )
// }
return result
} catch (error) {
log.error('Error Getting Discover Articles', error)
return {
__typename: 'AddDiscoverFeedError',
errorCodes: [AddDiscoverFeedErrorCode.Unauthorized],
}
}
})

90
packages/api/src/resolvers/discover_feeds/articles/add.ts Normal file
View file

@ -0,0 +1,90 @@
import { v4 } from 'uuid'
import { appDataSource } from '../../../data_source'
import {
InputMaybe,
MutationSaveDiscoverArticleArgs,
SaveDiscoverArticleError,
SaveDiscoverArticleErrorCode,
SaveDiscoverArticleSuccess,
SaveSuccess,
} from '../../../generated/graphql'
import { userRepository } from '../../../repository/user'
import { saveUrl } from '../../../services/save_url'
import { authorized } from '../../../utils/gql-utils'
export const saveDiscoverArticleResolver = authorized<
SaveDiscoverArticleSuccess,
SaveDiscoverArticleError,
MutationSaveDiscoverArticleArgs
>(
async (
_,
{ input: { discoverArticleId, timezone, locale } },
{ uid, log }
) => {
try {
const user = await userRepository.findById(uid)
if (!user) {
return {
__typename: 'SaveDiscoverArticleError',
errorCodes: [SaveDiscoverArticleErrorCode.Unauthorized],
}
}
const { rows: discoverArticles } = (await appDataSource.query(
`SELECT url FROM omnivore.discover_feed_articles WHERE id=$1`,
[discoverArticleId]
)) as {
rows: {
url: string
}[]
}
if (discoverArticles.length != 1) {
return {
__typename: 'SaveDiscoverArticleError',
errorCodes: [SaveDiscoverArticleErrorCode.NotFound],
}
}
const url = discoverArticles[0].url
const savedArticle = await saveUrl(
{
url,
source: 'add-link',
clientRequestId: v4(),
locale: locale as InputMaybe<string>,
timezone: timezone as InputMaybe<string>,
},
user
)
if (savedArticle.__typename == 'SaveError') {
return {
__typename: 'SaveDiscoverArticleError',
errorCodes: [SaveDiscoverArticleErrorCode.BadRequest],
}
}
const saveSuccess = savedArticle as SaveSuccess
await appDataSource.query(
`insert into omnivore.discover_feed_save_link (discover_article_id, user_id, article_save_id, article_save_url) VALUES ($1, $2, $3, $4) ON CONFLICT ON CONSTRAINT user_discover_feed_link DO UPDATE SET (article_save_id, article_save_url, deleted) = ($3, $4, false);`,
[discoverArticleId, uid, saveSuccess.clientRequestId, saveSuccess.url]
)
return {
__typename: 'SaveDiscoverArticleSuccess',
url: saveSuccess.url,
saveId: saveSuccess.clientRequestId,
}
} catch (error) {
log.error('Error Saving Article', error)
return {
__typename: 'SaveDiscoverArticleError',
errorCodes: [SaveDiscoverArticleErrorCode.Unauthorized],
}
}
}
)

68
packages/api/src/resolvers/discover_feeds/articles/delete.ts Normal file
View file

@ -0,0 +1,68 @@
import { appDataSource } from '../../../data_source'
import { LibraryItemState } from '../../../entity/library_item'
import {
DeleteDiscoverArticleError,
DeleteDiscoverArticleErrorCode,
DeleteDiscoverArticleSuccess,
MutationDeleteDiscoverArticleArgs,
} from '../../../generated/graphql'
import { userRepository } from '../../../repository/user'
import { updateLibraryItem } from '../../../services/library_item'
import { authorized } from '../../../utils/gql-utils'
export const deleteDiscoverArticleResolver = authorized<
DeleteDiscoverArticleSuccess,
DeleteDiscoverArticleError,
MutationDeleteDiscoverArticleArgs
>(async (_, { input: { discoverArticleId } }, { uid, log, pubsub }) => {
try {
const user = await userRepository.findById(uid)
if (!user) {
return {
__typename: 'DeleteDiscoverArticleError',
errorCodes: [DeleteDiscoverArticleErrorCode.Unauthorized],
}
}
const { rows: discoverArticles } = (await appDataSource.query(
`SELECT article_save_id FROM omnivore.discover_feed_save_link WHERE discover_article_id=$1 and user_id=$2`,
[discoverArticleId, uid]
)) as {
rows: { article_save_id: string }[]
}
if (discoverArticles.length != 1) {
return {
__typename: 'DeleteDiscoverArticleError',
errorCodes: [DeleteDiscoverArticleErrorCode.NotFound],
}
}
await appDataSource.query(
`UPDATE omnivore.discover_feed_save_link set deleted = true WHERE discover_article_id=$1 and user_id=$2`,
[discoverArticleId, uid]
)
await updateLibraryItem(
discoverArticles[0].article_save_id,
{
state: LibraryItemState.Deleted,
deletedAt: new Date(),
},
uid,
pubsub
)
return {
__typename: 'DeleteDiscoverArticleSuccess',
id: discoverArticleId,
}
} catch (error) {
log.error('Error Deleting Article', error)
return {
__typename: 'DeleteDiscoverArticleError',
errorCodes: [DeleteDiscoverArticleErrorCode.Unauthorized],
}
}
})

188
packages/api/src/resolvers/discover_feeds/articles/get.ts Normal file
View file

@ -0,0 +1,188 @@
import { appDataSource } from '../../../data_source'
import {
GetDiscoverFeedArticleError,
GetDiscoverFeedArticleErrorCode,
GetDiscoverFeedArticleSuccess,
QueryGetDiscoverFeedArticlesArgs,
} from '../../../generated/graphql'
import { authorized } from '../../../utils/gql-utils'
const COMMUNITY_FEED_ID = '8217d320-aa5a-11ee-bbfe-a7cde356f524'
type DiscoverFeedArticleDBRows = {
rows: {
id: string
feed: string
title: string
slug: string
url: string
author: string
image: string
published_at: Date
description: string
saves: number
article_save_id: string | undefined
article_save_url: string | undefined
}[]
}
const getPopularTopics = (
uid: string,
after: string,
amt: number,
feedId: string | null = null
): Promise<DiscoverFeedArticleDBRows> => {
const params = [uid, amt + 1, after]
if (feedId) {
params.push(feedId)
}
return appDataSource.query(
`
SELECT id, title, feed_id as feed, slug, description, url, author, image, published_at, COALESCE(sl.count / (EXTRACT(EPOCH FROM (NOW() - published_at)) / 3600 / 24), 0) as popularity_score, article_save_id, article_save_url
FROM omnivore.discover_feed_articles
LEFT JOIN (SELECT discover_article_id as article_id, count(*) as count FROM omnivore.discover_feed_save_link group by discover_article_id) sl on id=sl.article_id
LEFT JOIN (SELECT discover_article_id, article_save_id, article_save_url FROM omnivore.discover_feed_save_link WHERE user_id=$1 and deleted = false) su on id=su.discover_article_id
WHERE COALESCE(sl.count / (EXTRACT(EPOCH FROM (NOW() - published_at)) / 3600 / 24), 0) > 0.0
AND (feed_id in (SELECT feed_id FROM omnivore.discover_feed_subscription WHERE user_id = $1) OR feed_id = '${COMMUNITY_FEED_ID}') ${
feedId != null ? `AND feed_id = $4` : ''
}
ORDER BY popularity_score DESC
LIMIT $2 OFFSET $3
`,
params
) as Promise<DiscoverFeedArticleDBRows>
}
const getAllTopics = (
uid: string,
after: string,
amt: number,
feedId: string | null = null
): Promise<DiscoverFeedArticleDBRows> => {
const params = [uid, amt + 1, after]
if (feedId) {
params.push(feedId)
}
return appDataSource.query(
`
SELECT id, title, feed_id as feed, slug, description, url, author, image, published_at, article_save_id, article_save_url
FROM omnivore.discover_feed_articles
LEFT JOIN (SELECT discover_article_id, article_save_id, article_save_url FROM omnivore.discover_feed_save_link WHERE user_id=$1 and deleted = false) su on id=su.discover_article_id
WHERE (feed_id in (SELECT feed_id FROM omnivore.discover_feed_subscription WHERE user_id = $1) OR feed_id = '${COMMUNITY_FEED_ID}')
${feedId != null ? `AND feed_id = $4` : ''}
ORDER BY published_at DESC
LIMIT $2 OFFSET $3
`,
params
) as Promise<DiscoverFeedArticleDBRows>
}
const getTopicInformation = (
discoverTopicId: string,
uid: string,
after: string,
amt: number,
feedId: string | null = null
): Promise<DiscoverFeedArticleDBRows> => {
const params = [uid, discoverTopicId, amt + 1, Number(after)]
if (feedId) {
params.push(feedId)
}
return appDataSource.query(
`SELECT id, title, feed_id as feed, slug, description, url, author, image, published_at, article_save_id, article_save_url
FROM omnivore.discover_feed_articles
INNER JOIN (SELECT discover_feed_article_id FROM omnivore.discover_feed_article_topic_link WHERE discover_topic_name=$2) topic on topic.discover_feed_article_id=id
LEFT JOIN (SELECT discover_article_id, article_save_id, article_save_url FROM omnivore.discover_feed_save_link WHERE user_id=$1 and deleted = false) su on id=su.discover_article_id
WHERE (feed_id in (SELECT feed_id FROM omnivore.discover_feed_subscription WHERE user_id = $1) OR feed_id = '${COMMUNITY_FEED_ID}')
${feedId != null ? `AND feed_id = $5` : ''}
ORDER BY published_at DESC
LIMIT $3 OFFSET $4
`,
params
) as Promise<DiscoverFeedArticleDBRows>
}
export const getDiscoverFeedArticlesResolver = authorized<
GetDiscoverFeedArticleSuccess,
GetDiscoverFeedArticleError,
QueryGetDiscoverFeedArticlesArgs
>(async (_, { discoverTopicId, feedId, first, after }, { uid, log }) => {
try {
const startCursor: string = after || ''
const firstAmnt = Math.min(first || 10, 100) // limit to 100 items
const { rows: topics } = (await appDataSource.query(
`SELECT * FROM "omnivore"."discover_topics" WHERE "name" = $1`,
[discoverTopicId]
)) as { rows: unknown[] }
if (topics.length == 0) {
return {
__typename: 'GetDiscoverFeedArticleError',
errorCodes: [GetDiscoverFeedArticleErrorCode.Unauthorized], // TODO - no.
}
}
let discoverArticles: DiscoverFeedArticleDBRows = { rows: [] }
if (discoverTopicId === 'Popular') {
discoverArticles = await getPopularTopics(
uid,
startCursor,
firstAmnt,
feedId ?? null
)
} else if (discoverTopicId === 'All') {
discoverArticles = await getAllTopics(
uid,
startCursor,
firstAmnt,
feedId ?? null
)
} else {
discoverArticles = await getTopicInformation(
discoverTopicId,
uid,
startCursor,
firstAmnt,
feedId ?? null
)
}
return {
__typename: 'GetDiscoverFeedArticleSuccess',
discoverArticles: discoverArticles.rows.slice(0, firstAmnt).map((it) => ({
author: it.author,
id: it.id,
feed: it.feed,
slug: it.slug,
publishedDate: it.published_at,
description: it.description,
url: it.url,
title: it.title,
image: it.image,
saves: it.saves,
savedLinkUrl: it.article_save_url,
savedId: it.article_save_id,
__typename: 'DiscoverFeedArticle',
siteName: it.url,
})),
pageInfo: {
endCursor: `${
Number(startCursor) +
Math.min(discoverArticles.rows.length, firstAmnt)
}`,
hasNextPage: discoverArticles.rows.length > firstAmnt,
hasPreviousPage: Number(startCursor) != 0,
startCursor: Number(startCursor).toString(),
totalCount: Math.min(discoverArticles.rows.length, firstAmnt),
},
}
} catch (error) {
log.error('Error Getting Discover Feed Articles', error)
return {
__typename: 'GetDiscoverFeedArticleError',
errorCodes: [GetDiscoverFeedArticleErrorCode.Unauthorized],
}
}
})

52
packages/api/src/resolvers/discover_feeds/delete.ts Normal file
View file

@ -0,0 +1,52 @@
import { appDataSource } from '../../data_source'
import {
DeleteDiscoverFeedError,
DeleteDiscoverFeedErrorCode,
DeleteDiscoverFeedSuccess,
MutationDeleteDiscoverFeedArgs,
} from '../../generated/graphql'
import { authorized } from '../../utils/gql-utils'
export const deleteDiscoverFeedsResolver = authorized<
DeleteDiscoverFeedSuccess,
DeleteDiscoverFeedError,
MutationDeleteDiscoverFeedArgs
>(async (_, { input: { feedId } }, { uid, log }) => {
try {
// Ensure that it actually exists for the user.
const feeds = (await appDataSource.query(
`SELECT * FROM omnivore.discover_feed_subscription sub
WHERE sub.user_id = $1 and sub.feed_id = $2`,
[uid, feedId]
)) as {
rows: {
feed_id: string
}[]
}
if (feeds.rows.length == 0) {
return {
__typename: 'DeleteDiscoverFeedError',
errorCodes: [DeleteDiscoverFeedErrorCode.NotFound],
}
}
await appDataSource.query(
`DELETE FROM omnivore.discover_feed_subscription sub
WHERE sub.user_id = $1 and sub.feed_id = $2`,
[uid, feedId]
)
return {
__typename: 'DeleteDiscoverFeedSuccess',
id: feedId,
}
} catch (error) {
log.error('Error Getting Discover Feed Subscriptions', error)
return {
__typename: 'DeleteDiscoverFeedError',
errorCodes: [DeleteDiscoverFeedErrorCode.Unauthorized],
}
}
})

52
packages/api/src/resolvers/discover_feeds/edit.ts Normal file
View file

@ -0,0 +1,52 @@
import { appDataSource } from '../../data_source'
import {
EditDiscoverFeedError,
EditDiscoverFeedErrorCode,
EditDiscoverFeedSuccess,
MutationEditDiscoverFeedArgs,
} from '../../generated/graphql'
import { authorized } from '../../utils/gql-utils'
export const editDiscoverFeedsResolver = authorized<
EditDiscoverFeedSuccess,
EditDiscoverFeedError,
MutationEditDiscoverFeedArgs
>(async (_, { input: { feedId, name } }, { uid, log }) => {
try {
// Ensure that it actually exists for the user.
const feeds = (await appDataSource.query(
`SELECT * FROM omnivore.discover_feed_subscription sub
WHERE sub.user_id = $1 and sub.feed_id = $2`,
[uid, feedId]
)) as {
rows: {
feed_id: string
}[]
}
if (feeds.rows.length == 0) {
return {
__typename: 'EditDiscoverFeedError',
errorCodes: [EditDiscoverFeedErrorCode.NotFound],
}
}
await appDataSource.query(
`UPDATE omnivore.discover_feed_subscription SET visible_name = $1
WHERE user_id = $2 and feed_id = $3`,
[name, uid, feedId]
)
return {
__typename: 'EditDiscoverFeedSuccess',
id: feedId,
}
} catch (error) {
log.error('Error Updating Discover Feed Subscriptions', error)
return {
__typename: 'EditDiscoverFeedError',
errorCodes: [EditDiscoverFeedErrorCode.Unauthorized],
}
}
})

36
packages/api/src/resolvers/discover_feeds/get.ts Normal file
View file

@ -0,0 +1,36 @@
import { appDataSource } from '../../data_source'
import {
DiscoverFeed,
DiscoverFeedError,
DiscoverFeedErrorCode,
DiscoverFeedSuccess,
} from '../../generated/graphql'
import { authorized } from '../../utils/gql-utils'
export const getDiscoverFeedsResolver = authorized<
DiscoverFeedSuccess,
DiscoverFeedError
>(async (_, _args, { uid, log }) => {
try {
const existingFeed = (await appDataSource.query(
`SELECT *, COALESCE(visible_name, title) as "visibleName" FROM omnivore.discover_feed_subscription sub
INNER JOIN omnivore.discover_feed feed on sub.feed_id=id
WHERE sub.user_id = $1`,
[uid]
)) as {
rows: DiscoverFeed[]
}
return {
__typename: 'DiscoverFeedSuccess',
feeds: existingFeed.rows || [],
}
} catch (error) {
log.error('Error Getting Discover Feed Subscriptions', error)
return {
__typename: 'DiscoverFeedError',
errorCodes: [DiscoverFeedErrorCode.Unauthorized],
}
}
})

7
packages/api/src/resolvers/discover_feeds/index.ts Normal file
View file

@ -0,0 +1,7 @@
export * from './add'
export * from './articles/get'
export * from './get'
export * from './articles/delete'
export * from './delete'
export * from './articles/add'
export * from './edit'

55
packages/api/src/resolvers/function_resolvers.ts
View file

@ -22,6 +22,8 @@ import {
SearchItem,
User,
} from '../generated/graphql'
import { getAISummary } from '../services/ai-summaries'
import { findUserFeatures } from '../services/features'
import { findHighlightsByLibraryItemId } from '../services/highlights'
import { findLabelsByLibraryItemId } from '../services/labels'
import { findRecommendationsByLibraryItemId } from '../services/recommendation'
@ -39,6 +41,15 @@ import {
generateUploadFilePathName,
} from '../utils/uploads'
import { emptyTrashResolver, fetchContentResolver } from './article'
import {
addDiscoverFeedResolver,
deleteDiscoverArticleResolver,
deleteDiscoverFeedsResolver,
editDiscoverFeedsResolver,
getDiscoverFeedArticlesResolver,
getDiscoverFeedsResolver,
saveDiscoverArticleResolver,
} from './discover_feeds'
import { optInFeatureResolver } from './features'
import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
import {
@ -63,6 +74,7 @@ import {
deleteRuleResolver,
deleteWebhookResolver,
deviceTokensResolver,
exportToIntegrationResolver,
feedsResolver,
filtersResolver,
generateApiKeyResolver,
@ -79,6 +91,7 @@ import {
googleSignupResolver,
groupsResolver,
importFromIntegrationResolver,
integrationResolver,
integrationsResolver,
joinGroupResolver,
labelsResolver,
@ -293,13 +306,21 @@ export const functionResolvers = {
updateSubscription: updateSubscriptionResolver,
updateFilter: updateFilterResolver,
updateEmail: updateEmailResolver,
saveDiscoverArticle: saveDiscoverArticleResolver,
deleteDiscoverArticle: deleteDiscoverArticleResolver,
moveToFolder: moveToFolderResolver,
updateNewsletterEmail: updateNewsletterEmailResolver,
addDiscoverFeed: addDiscoverFeedResolver,
deleteDiscoverFeed: deleteDiscoverFeedsResolver,
editDiscoverFeed: editDiscoverFeedsResolver,
emptyTrash: emptyTrashResolver,
fetchContent: fetchContentResolver,
exportToIntegration: exportToIntegrationResolver,
},
Query: {
me: getMeUserResolver,
getDiscoverFeedArticles: getDiscoverFeedArticlesResolver,
discoverFeeds: getDiscoverFeedsResolver,
user: getUserResolver,
users: getAllUsersResolver,
validateUsername: validateUsernameResolver,
@ -330,6 +351,7 @@ export const functionResolvers = {
recentEmails: recentEmailsResolver,
feeds: feedsResolver,
scanFeeds: scanFeedsResolver,
integration: integrationResolver,
},
User: {
async intercomHash(
@ -346,6 +368,28 @@ export const functionResolvers = {
}
return undefined
},
async featureList(
_: User,
__: Record<string, unknown>,
ctx: WithDataSourcesContext
) {
if (!ctx.claims?.uid) {
return undefined
}
return findUserFeatures(ctx.claims.uid)
},
async features(
user: User,
__: Record<string, unknown>,
ctx: WithDataSourcesContext
) {
if (!ctx.claims?.uid) {
return undefined
}
return (await findUserFeatures(ctx.claims.uid)).map((f) => f.name)
},
},
Article: {
async url(article: Article, _: unknown, ctx: WithDataSourcesContext) {
@ -485,6 +529,15 @@ export const functionResolvers = {
return []
},
async aiSummary(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) {
return (
await getAISummary({
userId: ctx.uid,
libraryItemId: item.id,
idx: 'latest',
})
)?.summary
},
async highlights(
item: {
id: string
@ -625,4 +678,6 @@ export const functionResolvers = {
...resultResolveTypeResolver('UpdateNewsletterEmail'),
...resultResolveTypeResolver('EmptyTrash'),
...resultResolveTypeResolver('FetchContent'),
...resultResolveTypeResolver('Integration'),
...resultResolveTypeResolver('ExportToIntegration'),
}

353
packages/api/src/resolvers/integrations/index.ts
View file

@ -9,22 +9,31 @@ import {
DeleteIntegrationError,
DeleteIntegrationErrorCode,
DeleteIntegrationSuccess,
ExportToIntegrationError,
ExportToIntegrationErrorCode,
ExportToIntegrationSuccess,
ImportFromIntegrationError,
ImportFromIntegrationErrorCode,
ImportFromIntegrationSuccess,
IntegrationError,
IntegrationErrorCode,
IntegrationsError,
IntegrationsErrorCode,
IntegrationsSuccess,
IntegrationSuccess,
MutationDeleteIntegrationArgs,
MutationExportToIntegrationArgs,
MutationImportFromIntegrationArgs,
MutationSetIntegrationArgs,
QueryIntegrationArgs,
SetIntegrationError,
SetIntegrationErrorCode,
SetIntegrationSuccess,
TaskState,
} from '../../generated/graphql'
import { createIntegrationToken } from '../../routers/auth/jwt_helpers'
import {
findIntegration,
findIntegrationByName,
findIntegrations,
getIntegrationClient,
removeIntegration,
@ -34,7 +43,7 @@ import {
import { analytics } from '../../utils/analytics'
import {
deleteTask,
enqueueExportAllItems,
enqueueExportToIntegration,
enqueueImportFromIntegration,
} from '../../utils/createTask'
import { authorized } from '../../utils/gql-utils'
@ -43,117 +52,94 @@ export const setIntegrationResolver = authorized<
SetIntegrationSuccess,
SetIntegrationError,
MutationSetIntegrationArgs
>(async (_, { input }, { uid, log }) => {
try {
const integrationToSave: DeepPartial<Integration> = {
...input,
user: { id: uid },
id: input.id || undefined,
type: input.type || IntegrationType.Export,
syncedAt: input.syncedAt ? new Date(input.syncedAt) : undefined,
importItemState:
input.type === IntegrationType.Import
? input.importItemState || ImportItemState.Unarchived // default to unarchived
: undefined,
}
if (input.id) {
// Update
const existingIntegration = await findIntegration({ id: input.id }, uid)
if (!existingIntegration) {
return {
errorCodes: [SetIntegrationErrorCode.NotFound],
}
>(async (_, { input }, { uid }) => {
const integrationToSave: DeepPartial<Integration> = {
...input,
user: { id: uid },
id: input.id || undefined,
type: input.type || IntegrationType.Export,
syncedAt: input.syncedAt ? new Date(input.syncedAt) : undefined,
importItemState:
input.type === IntegrationType.Import
? input.importItemState || ImportItemState.Unarchived // default to unarchived
: undefined,
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
settings: input.settings,
}
if (input.id) {
// Update
const existingIntegration = await findIntegration({ id: input.id }, uid)
if (!existingIntegration) {
return {
errorCodes: [SetIntegrationErrorCode.NotFound],
}
}
integrationToSave.id = existingIntegration.id
integrationToSave.taskName = existingIntegration.taskName
} else {
// Create
const integrationService = getIntegrationClient(input.name)
// authorize and get access token
const token = await integrationService.accessToken(input.token)
if (!token) {
return {
errorCodes: [SetIntegrationErrorCode.InvalidToken],
}
integrationToSave.id = existingIntegration.id
integrationToSave.taskName = existingIntegration.taskName
} else {
// Create
const integrationService = getIntegrationClient(input.name, input.token)
// authorize and get access token
const token = await integrationService.accessToken()
if (!token) {
return {
errorCodes: [SetIntegrationErrorCode.InvalidToken],
}
integrationToSave.token = token
}
integrationToSave.token = token
}
// save integration
const integration = await saveIntegration(integrationToSave, uid)
// save integration
const integration = await saveIntegration(integrationToSave, uid)
if (integrationToSave.type === IntegrationType.Export && !input.id) {
const authToken = await createIntegrationToken({
uid,
token: integration.token,
})
if (!authToken) {
log.error('failed to create auth token', {
integrationId: integration.id,
})
return {
errorCodes: [SetIntegrationErrorCode.BadRequest],
}
}
if (integration.name.toLowerCase() === 'readwise') {
// create a task to export all the items for readwise temporarily
await enqueueExportToIntegration(integration.id, uid)
}
// create a task to sync all the pages if new integration or enable integration (export type)
await enqueueExportAllItems(integration.id, uid)
} else if (integrationToSave.taskName) {
// delete the task if disable integration and task exists
const result = await deleteTask(integrationToSave.taskName)
if (result) {
log.info('task deleted', integrationToSave.taskName)
}
analytics.capture({
distinctId: uid,
event: 'integration_set',
properties: {
id: integrationToSave.id,
env: env.server.apiEnv,
},
})
// update task name in integration
await updateIntegration(
integration.id,
{
taskName: null,
},
uid
)
integration.taskName = null
}
analytics.capture({
distinctId: uid,
event: 'integration_set',
properties: {
id: integrationToSave.id,
env: env.server.apiEnv,
},
})
return {
integration,
}
} catch (error) {
log.error(error)
return {
errorCodes: [SetIntegrationErrorCode.BadRequest],
}
return {
integration,
}
})
export const integrationsResolver = authorized<
IntegrationsSuccess,
IntegrationsError
>(async (_, __, { uid, log }) => {
try {
const integrations = await findIntegrations(uid)
>(async (_, __, { uid }) => {
const integrations = await findIntegrations(uid)
return {
integrations,
}
})
export const integrationResolver = authorized<
IntegrationSuccess,
IntegrationError,
QueryIntegrationArgs
>(async (_, { name }, { uid, log }) => {
const integration = await findIntegrationByName(name, uid)
if (!integration) {
log.error('integration not found', name)
return {
integrations,
errorCodes: [IntegrationErrorCode.NotFound],
}
} catch (error) {
log.error(error)
}
return {
errorCodes: [IntegrationsErrorCode.BadRequest],
}
return {
integration,
}
})
@ -164,42 +150,34 @@ export const deleteIntegrationResolver = authorized<
>(async (_, { id }, { claims: { uid }, log }) => {
log.info('deleteIntegrationResolver')
try {
const integration = await findIntegration({ id }, uid)
if (!integration) {
return {
errorCodes: [DeleteIntegrationErrorCode.NotFound],
}
}
if (integration.taskName) {
// delete the task if task exists
await deleteTask(integration.taskName)
log.info('task deleted', integration.taskName)
}
const deletedIntegration = await removeIntegration(integration, uid)
deletedIntegration.id = id
analytics.capture({
distinctId: uid,
event: 'integration_delete',
properties: {
integrationId: deletedIntegration.id,
env: env.server.apiEnv,
},
})
const integration = await findIntegration({ id }, uid)
if (!integration) {
return {
integration,
errorCodes: [DeleteIntegrationErrorCode.NotFound],
}
} catch (error) {
log.error(error)
}
return {
errorCodes: [DeleteIntegrationErrorCode.BadRequest],
}
if (integration.taskName) {
// delete the task if task exists
await deleteTask(integration.taskName)
log.info('task deleted', integration.taskName)
}
const deletedIntegration = await removeIntegration(integration, uid)
deletedIntegration.id = id
analytics.capture({
distinctId: uid,
event: 'integration_delete',
properties: {
integrationId: deletedIntegration.id,
env: env.server.apiEnv,
},
})
return {
integration,
}
})
@ -208,52 +186,93 @@ export const importFromIntegrationResolver = authorized<
ImportFromIntegrationError,
MutationImportFromIntegrationArgs
>(async (_, { integrationId }, { claims: { uid }, log }) => {
try {
const integration = await findIntegration({ id: integrationId }, uid)
if (!integration) {
return {
errorCodes: [ImportFromIntegrationErrorCode.Unauthorized],
}
}
const authToken = await createIntegrationToken({
uid: integration.user.id,
token: integration.token,
})
if (!authToken) {
return {
errorCodes: [ImportFromIntegrationErrorCode.BadRequest],
}
}
// create a task to import all the pages
const taskName = await enqueueImportFromIntegration(
integration.id,
integration.name,
integration.syncedAt?.getTime() || 0,
authToken,
integration.importItemState || ImportItemState.Unarchived
)
// update task name in integration
await updateIntegration(integration.id, { taskName }, uid)
analytics.capture({
distinctId: uid,
event: 'integration_import',
properties: {
integrationId,
},
})
const integration = await findIntegration({ id: integrationId }, uid)
if (!integration) {
return {
success: true,
errorCodes: [ImportFromIntegrationErrorCode.Unauthorized],
}
} catch (error) {
log.error(error)
}
const authToken = await createIntegrationToken({
uid: integration.user.id,
token: integration.token,
})
if (!authToken) {
return {
errorCodes: [ImportFromIntegrationErrorCode.BadRequest],
}
}
// create a task to import all the pages
const taskName = await enqueueImportFromIntegration(
integration.id,
integration.name,
integration.syncedAt?.getTime() || 0,
authToken,
integration.importItemState || ImportItemState.Unarchived
)
// update task name in integration
await updateIntegration(integration.id, { taskName }, uid)
analytics.capture({
distinctId: uid,
event: 'integration_import',
properties: {
integrationId,
},
})
return {
success: true,
}
})
export const exportToIntegrationResolver = authorized<
ExportToIntegrationSuccess,
ExportToIntegrationError,
MutationExportToIntegrationArgs
>(async (_, { integrationId }, { uid, log }) => {
const integration = await findIntegration({ id: integrationId }, uid)
if (!integration) {
log.error('integration not found', integrationId)
return {
errorCodes: [ExportToIntegrationErrorCode.Unauthorized],
}
}
// create a job to export all the items
const job = await enqueueExportToIntegration(integration.id, uid)
if (!job || !job.id) {
log.error('failed to create task', integrationId)
return {
errorCodes: [ExportToIntegrationErrorCode.FailedToCreateTask],
}
}
// update task name in integration
await updateIntegration(integration.id, { taskName: job.id }, uid)
analytics.capture({
distinctId: uid,
event: 'integration_export',
properties: {
integrationId,
},
})
return {
task: {
id: job.id,
name: job.name,
state: TaskState.Pending,
createdAt: new Date(job.timestamp),
progress: 0,
runningTime: 0,
cancellable: true,
},
}
})

4
packages/api/src/resolvers/recommendations/index.ts
View file

@ -171,7 +171,7 @@ export const recommendResolver = authorized<
member.user.id,
item.id,
{
group: { id: group.id },
group: { id: group.id, name: group.name },
note: input.note,
recommender: { id: uid },
createdAt: new Date(),
@ -278,11 +278,11 @@ export const recommendHighlightsResolver = authorized<
member.user.id,
item.id,
{
id: group.id,
note: input.note,
recommender: { id: uid },
createdAt: new Date(),
libraryItem: { id: item.id },
group: { id: group.id, name: group.name },
},
auth,
input.highlightIds

4
packages/api/src/resolvers/rules/index.ts
View file

@ -15,6 +15,7 @@ import {
} from '../../generated/graphql'
import { deleteRule } from '../../services/rules'
import { authorized } from '../../utils/gql-utils'
import { parseSearchQuery } from '../../utils/search'
export const setRuleResolver = authorized<
SetRuleSuccess,
@ -22,6 +23,9 @@ export const setRuleResolver = authorized<
MutationSetRuleArgs
>(async (_, { input }, { authTrx, uid, log }) => {
try {
// validate filter
parseSearchQuery(input.filter)
const rule = await authTrx((t) =>
t.getRepository(Rule).save({
...input,

4
packages/api/src/resolvers/subscriptions/index.ts
View file

@ -229,8 +229,8 @@ export const subscribeResolver = authorized<
...existingSubscription,
fetchContentType: input.fetchContentType
? (input.fetchContentType as FetchContentType)
: undefined,
folder: input.folder ?? undefined,
: existingSubscription.fetchContentType,
folder: input.folder ?? existingSubscription.folder,
isPrivate: input.isPrivate,
status: SubscriptionStatus.Active,
})

55
packages/api/src/routers/ai_summary_router.ts Normal file
View file

@ -0,0 +1,55 @@
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler'
import cors from 'cors'
import express from 'express'
import { userRepository } from '../repository/user'
import { getClaimsByToken } from '../utils/auth'
import { corsConfig } from '../utils/corsConfig'
import { getAISummary } from '../services/ai-summaries'
export function aiSummariesRouter() {
const router = express.Router()
// Get an indexed summary for an individual library item
router.get(
'/library-item/:libraryItemId/:idx',
cors<express.Request>(corsConfig),
async (req, res) => {
const token = req?.cookies?.auth || req?.headers?.authorization
const claims = await getClaimsByToken(token)
if (!claims) {
return res.status(401).send('UNAUTHORIZED')
}
const { uid } = claims
const user = await userRepository.findById(uid)
if (!user) {
return res.status(400).send('Bad Request')
}
const libraryItemId = req.params.libraryItemId
console.log('params: ', req.params)
if (!libraryItemId) {
return res.status(400).send('Bad request - no library item id provided')
}
const idx = req.params.idx
if (!idx) {
return res.status(400).send('Bad request - no idx provided')
}
const result = await getAISummary({
userId: user.id,
idx: req.params.idx,
libraryItemId: req.params.libraryItemId,
})
return res.send({
summary: result?.summary,
})
}
)
return router
}

45
packages/api/src/routers/auth/auth_router.ts
View file

@ -11,7 +11,6 @@ import axios from 'axios'
import cors from 'cors'
import type { Request, Response } from 'express'
import express from 'express'
import rateLimit from 'express-rate-limit'
import * as jwt from 'jsonwebtoken'
import url from 'url'
import { promisify } from 'util'
@ -36,6 +35,8 @@ import {
} from '../../utils/auth'
import { corsConfig } from '../../utils/corsConfig'
import { logger } from '../../utils/logger'
import { hourlyLimiter } from '../../utils/rate_limit'
import { verifyChallengeRecaptcha } from '../../utils/recaptcha'
import { createSsoToken, ssoRedirectURL } from '../../utils/sso'
import { handleAppleWebAuth } from './apple_auth'
import type { AuthProvider } from './auth_types'
@ -55,6 +56,7 @@ export interface SignupRequest {
username: string
bio?: string
pictureUrl?: string
recaptchaToken?: string
}
const signToken = promisify(jwt.sign)
@ -64,32 +66,31 @@ const cookieParams = {
maxAge: 365 * 24 * 60 * 60 * 1000,
}
const isURLPresent = (input: string): boolean => {
const urlRegex = /(https?:\/\/[^\s]+)/g
return urlRegex.test(input)
}
export const isValidSignupRequest = (obj: any): obj is SignupRequest => {
return (
'email' in obj &&
obj.email.trim().length > 0 &&
obj.email.trim().length < 512 && // email must not be empty
!isURLPresent(obj.email) &&
'password' in obj &&
obj.password.length >= 8 &&
obj.password.trim().length < 512 && // password must be at least 8 characters
'name' in obj &&
obj.name.trim().length > 0 &&
obj.name.trim().length < 512 && // name must not be empty
!isURLPresent(obj.name) &&
'username' in obj &&
obj.username.trim().length > 0 &&
obj.username.trim().length < 512 // username must not be empty
obj.username.trim().length < 512 && // username must not be empty
!isURLPresent(obj.username)
)
}
// The hourly limiter is used on the create account,
// and reset password endpoints
// this limits users to five operations per an hour
const hourlyLimiter = rateLimit({
windowMs: 60 * 60 * 1000,
max: 5,
skip: (req) => env.dev.isLocal,
})
export function authRouter() {
const router = express.Router()
@ -499,7 +500,27 @@ export function authRouter() {
`${env.client.url}/auth/email-signup?errorCodes=INVALID_CREDENTIALS`
)
}
const { email, password, name, username, bio, pictureUrl } = req.body
const {
email,
password,
name,
username,
bio,
pictureUrl,
recaptchaToken,
} = req.body
if (process.env.RECAPTCHA_CHALLENGE_SECRET_KEY) {
const verified =
recaptchaToken && (await verifyChallengeRecaptcha(recaptchaToken))
if (!verified) {
logger.info('recaptcha failed', recaptchaToken, verified)
return res.redirect(
`${env.client.url}/auth/email-signup?errorCodes=UNKNOWN`
)
}
}
// trim whitespace in email address
const trimmedEmail = email.trim()
try {

32
packages/api/src/routers/integration_router.ts
View file

@ -2,6 +2,7 @@ import axios from 'axios'
import cors from 'cors'
import express from 'express'
import { env } from '../env'
import { getIntegrationClient } from '../services/integrations'
import { getClaimsByToken } from '../utils/auth'
import { corsConfig } from '../utils/corsConfig'
import { logger } from '../utils/logger'
@ -10,10 +11,9 @@ export function integrationRouter() {
const router = express.Router()
// request token from pocket
router.post(
'/pocket/auth',
'/:name/auth',
cors<express.Request>(corsConfig),
async (req: express.Request, res: express.Response) => {
logger.info('pocket/request-token')
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const token = (req.cookies.auth as string) || req.headers.authorization
const claims = await getClaimsByToken(token)
@ -21,37 +21,19 @@ export function integrationRouter() {
return res.status(401).send('UNAUTHORIZED')
}
const consumerKey = env.pocket.consumerKey
const redirectUri = `${env.client.url}/settings/integrations`
const integrationClient = getIntegrationClient(req.params.name, '')
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
const state = req.body.state as string
try {
// make a POST request to Pocket to get a request token
const response = await axios.post<{ code: string }>(
'https://getpocket.com/v3/oauth/request',
{
consumer_key: consumerKey,
redirect_uri: redirectUri,
},
{
headers: {
'Content-Type': 'application/json',
'X-Accept': 'application/json',
},
}
)
const { code } = response.data
const redirectUri = await integrationClient.auth(state)
// redirect the user to Pocket to authorize the request token
res.redirect(
`https://getpocket.com/auth/authorize?request_token=${code}&redirect_uri=${redirectUri}${encodeURIComponent(
`?pocketToken=${code}&state=${state}`
)}`
)
res.redirect(redirectUri)
} catch (error) {
if (axios.isAxiosError(error)) {
logger.error(error.response)
} else {
logger.error('pocket/request-token exception:', error)
logger.error(error)
}
res.redirect(

65
packages/api/src/routers/svc/integrations.ts
View file

@ -1,65 +0,0 @@
/* eslint-disable @typescript-eslint/no-misused-promises */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import express from 'express'
import { Integration, IntegrationType } from '../../entity/integration'
import { readPushSubscription } from '../../pubsub'
import { getRepository } from '../../repository'
import { enqueueExportAllItems } from '../../utils/createTask'
import { logger } from '../../utils/logger'
import { createIntegrationToken } from '../auth/jwt_helpers'
export function integrationsServiceRouter() {
const router = express.Router()
router.post('/export', async (req, res) => {
logger.info('start to sync with integration')
try {
const { message: msgStr, expired } = readPushSubscription(req)
if (!msgStr) {
return res.status(200).send('Bad Request')
}
if (expired) {
logger.info('discarding expired message')
return res.status(200).send('Expired')
}
// find all active integrations
const integrations = await getRepository(Integration).find({
where: {
enabled: true,
type: IntegrationType.Export,
},
relations: ['user'],
})
// create a task to sync with each integration
await Promise.all(
integrations.map(async (integration) => {
const authToken = await createIntegrationToken({
uid: integration.user.id,
token: integration.token,
})
if (!authToken) {
logger.error('failed to create auth token', {
integrationId: integration.id,
})
return
}
return enqueueExportAllItems(integration.id, integration.user.id)
})
)
} catch (err) {
logger.error('sync with integrations failed', err)
return res.status(500).send(err)
}
res.status(200).send('OK')
})
return router
}

49
packages/api/src/routers/task_router.ts Normal file
View file

@ -0,0 +1,49 @@
import cors from 'cors'
import express from 'express'
import { Task, TaskState } from '../generated/graphql'
import { getJob, jobStateToTaskState } from '../queue-processor'
import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
import { corsConfig } from '../utils/corsConfig'
import { logger } from '../utils/logger'
export function taskRouter() {
const router = express.Router()
router.get('/:id', cors<express.Request>(corsConfig), async (req, res) => {
const token = getTokenByRequest(req)
const claims = await getClaimsByToken(token)
if (!claims) {
return res.status(401).send('UNAUTHORIZED')
}
try {
const job = await getJob(req.params.id)
if (!job || !job.id) {
res.status(404).send('Not Found')
return
}
const jobState = await job.getState()
const state = jobStateToTaskState(jobState)
const finishedAt = job.finishedOn ? job.finishedOn : Date.now()
const runningTime = job.processedOn ? finishedAt - job.processedOn : 0
const result: Task = {
id: job.id,
state,
createdAt: new Date(job.timestamp),
name: job.name,
runningTime,
progress: job.progress as number,
failedReason: state === TaskState.Failed ? job.failedReason : undefined,
}
res.send(result)
} catch (e) {
logger.error('failed to get task', e)
res.status(500)
}
})
return router
}

280
packages/api/src/schema.ts
View file

@ -88,6 +88,8 @@ const schema = gql`
email: String
source: String
intercomHash: String
features: [String]
featureList: [Feature!]
}
type Profile {
@ -353,6 +355,11 @@ const schema = gql`
note: String
}
enum DirectionalityType {
LTR
RTL
}
type Article {
id: ID!
title: String!
@ -398,6 +405,7 @@ const schema = gql`
wordsCount: Int
folder: String!
feedContent: String
directionality: DirectionalityType
}
# Query: article
@ -1647,6 +1655,8 @@ const schema = gql`
previewContentType: String
links: JSON
folder: String!
aiSummary: String
directionality: DirectionalityType
}
type SearchItemEdge {
@ -2006,6 +2016,7 @@ const schema = gql`
createdAt: Date!
updatedAt: Date
taskName: String
settings: JSON
}
enum IntegrationType {
@ -2041,6 +2052,7 @@ const schema = gql`
syncedAt: Date
importItemState: ImportItemState
taskName: String
settings: JSON
}
union IntegrationsResult = IntegrationsSuccess | IntegrationsError
@ -2141,6 +2153,7 @@ const schema = gql`
createdAt: Date!
updatedAt: Date
eventTypes: [RuleEventType!]!
failedAt: Date
}
type RuleAction {
@ -2673,6 +2686,112 @@ const schema = gql`
email: String!
}
# Query: GetDiscoverTopic
union GetDiscoverTopicResults =
GetDiscoverTopicSuccess
| GetDiscoverTopicError
enum GetDiscoverTopicErrorCode {
UNAUTHORIZED
}
type GetDiscoverTopicError {
errorCodes: [GetDiscoverTopicErrorCode!]!
}
type GetDiscoverTopicSuccess {
discoverTopics: [DiscoverTopic!]
}
type DiscoverTopic {
name: String!
description: String!
}
# Query: GetDiscoverFeedArticle
union GetDiscoverFeedArticleResults =
GetDiscoverFeedArticleSuccess
| GetDiscoverFeedArticleError
enum GetDiscoverFeedArticleErrorCode {
UNAUTHORIZED
NOT_FOUND
BAD_REQUEST
}
type GetDiscoverFeedArticleError {
errorCodes: [GetDiscoverFeedArticleErrorCode!]!
}
type GetDiscoverFeedArticleSuccess {
discoverArticles: [DiscoverFeedArticle]
pageInfo: PageInfo!
}
type DiscoverFeedArticle {
id: ID!
feed: String!
title: String!
url: String!
image: String
publishedDate: Date
description: String!
siteName: String
slug: String!
author: String
savedLinkUrl: String
savedId: String
}
# Mutation: SaveDiscoverArticle
input SaveDiscoverArticleInput {
discoverArticleId: ID!
locale: String
timezone: String
}
union SaveDiscoverArticleResult =
SaveDiscoverArticleSuccess
| SaveDiscoverArticleError
type SaveDiscoverArticleSuccess {
url: String!
saveId: String!
}
type SaveDiscoverArticleError {
errorCodes: [SaveDiscoverArticleErrorCode!]!
}
enum SaveDiscoverArticleErrorCode {
UNAUTHORIZED
BAD_REQUEST
NOT_FOUND
}
# Mutation: DeleteDiscoverArticle
input DeleteDiscoverArticleInput {
discoverArticleId: ID!
}
union DeleteDiscoverArticleResult =
DeleteDiscoverArticleSuccess
| DeleteDiscoverArticleError
type DeleteDiscoverArticleSuccess {
id: ID!
}
type DeleteDiscoverArticleError {
errorCodes: [DeleteDiscoverArticleErrorCode!]!
}
enum DeleteDiscoverArticleErrorCode {
UNAUTHORIZED
BAD_REQUEST
NOT_FOUND
}
input FeedsInput {
after: String
first: Int
@ -2802,6 +2921,146 @@ const schema = gql`
UNAUTHORIZED
}
type DiscoverFeed {
id: ID!
title: String!
link: String!
description: String
image: String
type: String!
visibleName: String
}
union DiscoverFeedResult = DiscoverFeedSuccess | DiscoverFeedError
type DiscoverFeedSuccess {
feeds: [DiscoverFeed]!
}
type DiscoverFeedError {
errorCodes: [DiscoverFeedErrorCode!]!
}
enum DiscoverFeedErrorCode {
UNAUTHORIZED
BAD_REQUEST
}
input AddDiscoverFeedInput {
url: String!
}
union AddDiscoverFeedResult = AddDiscoverFeedSuccess | AddDiscoverFeedError
type AddDiscoverFeedSuccess {
feed: DiscoverFeed!
}
type AddDiscoverFeedError {
errorCodes: [AddDiscoverFeedErrorCode!]!
}
enum AddDiscoverFeedErrorCode {
UNAUTHORIZED
BAD_REQUEST
CONFLICT
NOT_FOUND
}
union DeleteDiscoverFeedResult =
DeleteDiscoverFeedSuccess
| DeleteDiscoverFeedError
type DeleteDiscoverFeedSuccess {
id: String!
}
type DeleteDiscoverFeedError {
errorCodes: [DeleteDiscoverFeedErrorCode!]!
}
enum DeleteDiscoverFeedErrorCode {
UNAUTHORIZED
BAD_REQUEST
CONFLICT
NOT_FOUND
}
input DeleteDiscoverFeedInput {
feedId: ID!
}
union EditDiscoverFeedResult = EditDiscoverFeedSuccess | EditDiscoverFeedError
type EditDiscoverFeedSuccess {
id: ID!
}
type EditDiscoverFeedError {
errorCodes: [EditDiscoverFeedErrorCode!]!
}
enum EditDiscoverFeedErrorCode {
UNAUTHORIZED
BAD_REQUEST
NOT_FOUND
}
input EditDiscoverFeedInput {
feedId: ID!
name: String!
}
union IntegrationResult = IntegrationSuccess | IntegrationError
type IntegrationSuccess {
integration: Integration!
}
type IntegrationError {
errorCodes: [IntegrationErrorCode!]!
}
enum IntegrationErrorCode {
NOT_FOUND
}
union ExportToIntegrationResult =
ExportToIntegrationSuccess
| ExportToIntegrationError
type ExportToIntegrationSuccess {
task: Task!
}
type Task {
id: ID!
name: String!
state: TaskState!
createdAt: Date!
runningTime: Int # in milliseconds
cancellable: Boolean
progress: Float
failedReason: String
}
enum TaskState {
PENDING
RUNNING
SUCCEEDED
FAILED
CANCELLED
}
type ExportToIntegrationError {
errorCodes: [ExportToIntegrationErrorCode!]!
}
enum ExportToIntegrationErrorCode {
UNAUTHORIZED
FAILED_TO_CREATE_TASK
}
# Mutations
type Mutation {
googleLogin(input: GoogleLoginInput!): LoginResult!
@ -2869,6 +3128,12 @@ const schema = gql`
unsubscribe(name: String!, subscriptionId: ID): UnsubscribeResult!
subscribe(input: SubscribeInput!): SubscribeResult!
addPopularRead(name: String!): AddPopularReadResult!
saveDiscoverArticle(
input: SaveDiscoverArticleInput!
): SaveDiscoverArticleResult!
deleteDiscoverArticle(
input: DeleteDiscoverArticleInput!
): DeleteDiscoverArticleResult!
setWebhook(input: SetWebhookInput!): SetWebhookResult!
deleteWebhook(id: ID!): DeleteWebhookResult!
revokeApiKey(id: ID!): RevokeApiKeyResult!
@ -2904,6 +3169,7 @@ const schema = gql`
arguments: JSON # additional arguments for the action
): BulkActionResult!
importFromIntegration(integrationId: ID!): ImportFromIntegrationResult!
exportToIntegration(integrationId: ID!): ExportToIntegrationResult!
setFavoriteArticle(id: ID!): SetFavoriteArticleResult!
updateSubscription(
input: UpdateSubscriptionInput!
@ -2913,6 +3179,11 @@ const schema = gql`
updateNewsletterEmail(
input: UpdateNewsletterEmailInput!
): UpdateNewsletterEmailResult!
addDiscoverFeed(input: AddDiscoverFeedInput!): AddDiscoverFeedResult!
deleteDiscoverFeed(
input: DeleteDiscoverFeedInput!
): DeleteDiscoverFeedResult!
editDiscoverFeed(input: EditDiscoverFeedInput!): EditDiscoverFeedResult!
emptyTrash: EmptyTrashResult!
}
@ -2950,6 +3221,13 @@ const schema = gql`
includeContent: Boolean
format: String
): SearchResult!
getDiscoverFeedArticles(
discoverTopicId: String!
feedId: ID
after: String
first: Int
): GetDiscoverFeedArticleResults!
discoverTopics: GetDiscoverTopicResults!
subscriptions(
sort: SortParams
type: SubscriptionType
@ -2966,6 +3244,7 @@ const schema = gql`
sort: SortParams
folder: String
): UpdatesSinceResult!
integration(name: String!): IntegrationResult!
integrations: IntegrationsResult!
recentSearches: RecentSearchesResult!
rules(enabled: Boolean): RulesResult!
@ -2974,6 +3253,7 @@ const schema = gql`
groups: GroupsResult!
recentEmails: RecentEmailsResult!
feeds(input: FeedsInput!): FeedsResult!
discoverFeeds: DiscoverFeedResult!
scanFeeds(input: ScanFeedsInput!): ScanFeedsResult!
}
`

54
packages/api/src/server.ts
View file

@ -10,7 +10,6 @@ import cookieParser from 'cookie-parser'
import express, { Express } from 'express'
import * as httpContext from 'express-http-context2'
import promBundle from 'express-prom-bundle'
import rateLimit from 'express-rate-limit'
import { createServer, Server } from 'http'
import * as prom from 'prom-client'
import { config, loggers } from 'winston'
@ -18,6 +17,7 @@ import { makeApolloServer } from './apollo'
import { appDataSource } from './data_source'
import { env } from './env'
import { redisDataSource } from './redis_data_source'
import { aiSummariesRouter } from './routers/ai_summary_router'
import { articleRouter } from './routers/article_router'
import { authRouter } from './routers/auth/auth_router'
import { mobileAuthRouter } from './routers/auth/mobile/mobile_auth_router'
@ -29,7 +29,6 @@ import { contentServiceRouter } from './routers/svc/content'
import { emailsServiceRouter } from './routers/svc/emails'
import { emailAttachmentRouter } from './routers/svc/email_attachment'
import { followingServiceRouter } from './routers/svc/following'
import { integrationsServiceRouter } from './routers/svc/integrations'
import { linkServiceRouter } from './routers/svc/links'
import { newsletterServiceRouter } from './routers/svc/newsletters'
// import { remindersServiceRouter } from './routers/svc/reminders'
@ -37,17 +36,14 @@ import { rssFeedRouter } from './routers/svc/rss_feed'
import { uploadServiceRouter } from './routers/svc/upload'
import { userServiceRouter } from './routers/svc/user'
import { webhooksServiceRouter } from './routers/svc/webhooks'
import { taskRouter } from './routers/task_router'
import { textToSpeechRouter } from './routers/text_to_speech'
import { userRouter } from './routers/user_router'
import { sentryConfig } from './sentry'
import { analytics } from './utils/analytics'
import {
getClaimsByToken,
getTokenByRequest,
isSystemRequest,
} from './utils/auth'
import { corsConfig } from './utils/corsConfig'
import { buildLogger, buildLoggerTransport } from './utils/logger'
import { buildLogger, buildLoggerTransport, logger } from './utils/logger'
import { apiLimiter, authLimiter } from './utils/rate_limit'
const PORT = process.env.PORT || 4000
@ -71,27 +67,6 @@ export const createApp = (): {
// set to true if behind a reverse proxy/load balancer
app.set('trust proxy', env.server.trustProxy)
const apiLimiter = rateLimit({
windowMs: 60 * 1000, // 1 minute
max: async (req) => {
// 100 RPM for an authenticated request, 15 for a non-authenticated request
const token = getTokenByRequest(req)
try {
const claims = await getClaimsByToken(token)
return claims ? 60 : 15
} catch (e) {
console.log('non-authenticated request')
return 15
}
},
keyGenerator: (req) => {
return getTokenByRequest(req) || req.ip
},
// skip preflight requests and test requests and system requests
skip: (req) =>
req.method === 'OPTIONS' || env.dev.isLocal || isSystemRequest(req),
})
// Apply the rate limiting middleware to API calls only
app.use('/api/', apiLimiter)
@ -108,29 +83,22 @@ export const createApp = (): {
// respond healthy to auto-scaler.
app.get('/_ah/health', (req, res) => res.sendStatus(200))
// 5 RPM for auth requests
const authLimiter = rateLimit({
windowMs: 60 * 1000, // 1 minute
max: 5,
// skip preflight requests and test requests
skip: (req) => req.method === 'OPTIONS' || env.dev.isLocal,
})
app.use('/api/auth', authLimiter, authRouter())
app.use('/api/mobile-auth', authLimiter, mobileAuthRouter())
app.use('/api/page', pageRouter())
app.use('/api/user', userRouter())
app.use('/api/article', articleRouter())
app.use('/api/ai-summary', aiSummariesRouter())
app.use('/api/text-to-speech', textToSpeechRouter())
app.use('/api/notification', notificationRouter())
app.use('/api/integration', integrationRouter())
app.use('/api/tasks', taskRouter())
app.use('/svc/pubsub/content', contentServiceRouter())
app.use('/svc/pubsub/links', linkServiceRouter())
app.use('/svc/pubsub/newsletters', newsletterServiceRouter())
app.use('/svc/pubsub/emails', emailsServiceRouter())
app.use('/svc/pubsub/upload', uploadServiceRouter())
app.use('/svc/pubsub/webhooks', webhooksServiceRouter())
app.use('/svc/pubsub/integrations', integrationsServiceRouter())
app.use('/svc/pubsub/rss-feed', rssFeedRouter())
app.use('/svc/pubsub/user', userServiceRouter())
// app.use('/svc/reminders', remindersServiceRouter())
@ -245,6 +213,16 @@ const main = async (): Promise<void> => {
process.on('SIGINT', () => gracefulShutdown('SIGINT'))
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'))
process.on('uncaughtException', function (err) {
// Handle the error safely
logger.error('Uncaught exception', err)
})
process.on('unhandledRejection', (reason, promise) => {
// Handle the error safely
logger.error('Unhandled Rejection at: Promise', { promise, reason })
})
}
// only call main if the file was called from the CLI and wasn't required from another module

34
packages/api/src/services/ai-summaries.ts Normal file
View file

@ -0,0 +1,34 @@
import { AISummary } from '../entity/AISummary'
import { authTrx } from '../repository'
export const getAISummary = async (data: {
userId: string
idx: string
libraryItemId: string
}): Promise<AISummary | undefined> => {
const aiSummary = await authTrx(
async (t) => {
const repo = t.getRepository(AISummary)
if (data.idx == 'latest') {
return repo.findOne({
where: {
user: { id: data.userId },
libraryItem: { id: data.libraryItemId },
},
order: { createdAt: 'DESC' },
})
} else {
return repo.findOne({
where: {
id: data.idx,
user: { id: data.userId },
libraryItem: { id: data.libraryItemId },
},
})
}
},
undefined,
data.userId
)
return aiSummary ?? undefined
}

2
packages/api/src/services/cached_reading_position.ts
View file

@ -47,7 +47,7 @@ export const componentsForCachedReadingPositionKey = (
libraryItemID,
}
} catch (error) {
logger.log('exception getting cache key components', { cacheKey, error })
logger.error('exception getting cache key components', { cacheKey, error })
}
return undefined
}

55
packages/api/src/services/features.ts
View file

@ -6,8 +6,15 @@ import { env } from '../env'
import { getRepository } from '../repository'
import { logger } from '../utils/logger'
const MAX_ULTRA_REALISTIC_USERS = 1500
const MAX_YOUTUBE_TRANSCRIPT_USERS = 500
const MAX_NOTION_USERS = 1000
export enum FeatureName {
AISummaries = 'ai-summaries',
YouTubeTranscripts = 'youtube-transcripts',
UltraRealisticVoice = 'ultra-realistic-voice',
Notion = 'notion',
}
export const getFeatureName = (name: string): FeatureName | undefined => {
@ -18,17 +25,34 @@ export const optInFeature = async (
name: FeatureName,
uid: string
): Promise<Feature | undefined> => {
if (name === FeatureName.UltraRealisticVoice) {
return optInUltraRealisticVoice(uid)
switch (name) {
case FeatureName.UltraRealisticVoice:
return optInLimitedFeature(
FeatureName.UltraRealisticVoice,
uid,
MAX_ULTRA_REALISTIC_USERS
)
case FeatureName.YouTubeTranscripts:
return optInLimitedFeature(
FeatureName.YouTubeTranscripts,
uid,
MAX_YOUTUBE_TRANSCRIPT_USERS
)
case FeatureName.Notion:
return optInLimitedFeature(FeatureName.Notion, uid, MAX_NOTION_USERS)
default:
return undefined
}
return undefined
}
const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
const optInLimitedFeature = async (
featureName: string,
uid: string,
maxUsers: number
): Promise<Feature> => {
const feature = await getRepository(Feature).findOne({
where: {
name: FeatureName.UltraRealisticVoice,
name: featureName,
grantedAt: Not(IsNull()),
user: { id: uid },
},
@ -40,9 +64,7 @@ const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
return feature
}
const MAX_USERS = 1500
// opt in to feature for the first 1500 users
const optedInFeatures = (await appDataSource.query(
const optedInFeatures: Feature[] = (await appDataSource.query(
`insert into omnivore.features (user_id, name, granted_at)
select $1, $2, $3 from omnivore.features
where name = $2 and granted_at is not null
@ -50,7 +72,7 @@ const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
on conflict (user_id, name)
do update set granted_at = $3
returning *, granted_at as "grantedAt", created_at as "createdAt", updated_at as "updatedAt";`,
[uid, FeatureName.UltraRealisticVoice, new Date(), MAX_USERS]
[uid, featureName, new Date(), maxUsers]
)) as Feature[]
// if no new features were created then user has exceeded max users
@ -60,7 +82,7 @@ const optInUltraRealisticVoice = async (uid: string): Promise<Feature> => {
// create/update an opt-in record with null grantedAt
const optInRecord = {
user: { id: uid },
name: FeatureName.UltraRealisticVoice,
name: featureName,
grantedAt: null,
}
const result = await getRepository(Feature).upsert(optInRecord, [
@ -100,13 +122,20 @@ export const signFeatureToken = (
)
}
export const findFeatureByName = async (
export const findUserFeatures = async (userId: string) => {
return getRepository(Feature).findBy({
user: { id: userId },
})
}
export const findGrantedFeatureByName = async (
name: FeatureName,
userId: string
): Promise<Feature | null> => {
return await getRepository(Feature).findOneBy({
return getRepository(Feature).findOneBy({
name,
user: { id: userId },
grantedAt: Not(IsNull()),
})
}

2
packages/api/src/services/groups.ts
View file

@ -259,7 +259,7 @@ export const createLabelAndRuleForGroup = async (
},
],
// add a condition to check if the page is created
filter: `event:created recommendedBy:"${groupName}"`,
filter: `recommendedBy:"${groupName}"`,
})
await Promise.all([addLabelPromise, sendNotificationPromise])

Some files were not shown because too many files have changed in this diff Show more