Merge branch 'main' into feature/following-screen

android/Omnivore/app/build.gradle.kts

@@ -27,8 +27,8 @@ android {
         applicationId = "app.omnivore.omnivore"
         minSdk = 26
         targetSdk = 34
-        versionCode = 200004
-        versionName = "0.200.4"
+        versionCode = 2000050
+        versionName = "0.200.5"
 
         testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
         vectorDrawables {

apple/OmnivoreKit/Sources/App/Views/AI/FullScreenDigestView.swift

@@ -0,0 +1,303 @@
+import SwiftUI
+import Models
+import Services
+
+public class FullScreenDigestViewModel: ObservableObject {
+  @Published var isLoading = false
+  @Published var digest: DigestResult?
+
+  func load(dataService: DataService) async {
+    isLoading = true
+    if digest == nil {
+      do {
+        digest = try await dataService.getLatestDigest(timeoutInterval: 10)
+      } catch {
+        print("ERROR WITH DIGEST: ", error)
+      }
+    }
+    isLoading = false
+  }
+}
+
+struct DigestAudioItem: AudioItemProperties {
+  let audioItemType = Models.AudioItemType.digest
+  
+  var itemID = ""
+  
+  var title = "TITLE"
+  
+  var byline: String? = "byline"
+  
+  var imageURL: URL? = nil
+  
+  var language: String?
+  
+  var startIndex: Int = 0
+  var startOffset: Double = 0.0
+}
+
+@available(iOS 17.0, *)
+@MainActor
+struct FullScreenDigestView: View {
+  let viewModel: DigestViewModel = DigestViewModel()
+  let dataService: DataService
+  let audioController: AudioController
+
+  @Environment(\.dismiss) private var dismiss
+
+  let textBody = "In a significant political turn, the SOTU response faces unexpected collapse, " +
+  "marking a stark contrast to Trump's latest downturn, alongside an unprecedented " +
+  "surge in Biden's fundraising efforts as of 3/11/24, according to the TDPS Podcast. " +
+  "The analysis provides insights into the shifting dynamics of political support and " +
+  "the potential implications for future electoral strategies. Based on the information " +
+  "you provided, the video seems to discuss a recent event where former President " +
+  "Donald Trump made a controversial statement that shocked even his own audience. " +
+  "The video likely covers Trump's response to the State of the Union (SOTU) address " +
+  "and how it received negative feedback, possibly leading to a decline in his support " +
+  "or approval ratings. Additionally, it appears that the video touches upon a surge " +
+  "in fundraising for President Joe Biden's administration around March 11, 2024."
+
+  public init(dataService: DataService, audioController: AudioController) {
+    self.dataService = dataService
+    self.audioController = audioController
+  }
+
+  var body: some View {
+    // ZStack(alignment: Alignment(horizontal: .trailing, vertical: .top)) {
+    Group {
+      if viewModel.isLoading {
+        ProgressView()
+      } else {
+        itemBody
+          .task {
+            await viewModel.load(dataService: dataService)
+          }.onAppear {
+            self.audioController.play(itemAudioProperties: DigestAudioItem())
+          }
+      }
+    }            .navigationTitle("Omnivore digest")
+      .navigationBarTitleDisplayMode(.inline)
+
+//      HStack(alignment: .top) {
+//        Spacer()
+//        closeButton
+//      }
+//      .padding(20)
+    // }
+  }
+
+  var closeButton: some View {
+    Button(action: {
+      dismiss()
+    }, label: {
+      ZStack {
+        Circle()
+          .foregroundColor(Color.appGrayText)
+          .frame(width: 36, height: 36)
+          .opacity(0.1)
+
+        Image(systemName: "xmark")
+          .font(.appCallout)
+          .frame(width: 36, height: 36)
+      }
+    })
+    .buttonStyle(.plain)
+  }
+
+  @available(iOS 17.0, *)
+  var itemBody: some View {
+    VStack {
+      ScrollView(.vertical) {
+        VStack(spacing: 20) {
+          Text("SOTU response collapses, Trump hits new low, Biden fundraising explodes 3/11/24 TDPS Podcast")
+            .font(.title)
+          Text(textBody)
+            .font(.body)
+        }
+      }
+      // .scrollTargetBehavior(.paging)
+      // .ignoresSafeArea()
+      MiniPlayerViewer()
+        .padding(.top, 10)
+        .padding(.bottom, 40)
+        .background(Color.themeTabBarColor)
+        .onTapGesture {
+          // showExpandedAudioPlayer = true
+        }
+    }
+  }
+}
+
+@MainActor
+public class PreviewItemViewModel: ObservableObject {
+  let dataService: DataService
+  @Published var item: DigestItem
+  let showSwipeHint: Bool
+
+  @Published var isLoading = false
+  @Published var resultText: String?
+  @Published var promptDisplayText: String?
+
+  init(dataService: DataService, item: DigestItem, showSwipeHint: Bool) {
+    self.dataService = dataService
+    self.item = item
+    self.showSwipeHint = showSwipeHint
+  }
+
+  func loadResult() async {
+//    isLoading = true
+//    let taskId = try? await dataService.createAITask(
+//      extraText: extraText,
+//      libraryItemId: item?.id ?? "",
+//      promptName: "summarize-001"
+//    )
+//
+//    if let taskId = taskId {
+//      do {
+//        let fetchedText = try await dataService.pollAITask(jobId: taskId, timeoutInterval: 30)
+//        resultText = fetchedText
+//      } catch {
+//        print("ERROR WITH RESULT TEXT: ", error)
+//      }
+//    } else {
+//      print("NO TASK ID: ", taskId)
+//    }
+//    isLoading = false
+  }
+}
+
+@MainActor
+struct PreviewItemView: View {
+  @StateObject var viewModel: PreviewItemViewModel
+
+    var body: some View {
+      VStack(spacing: 10) {
+        HStack {
+          AsyncImage(url: viewModel.item.siteIcon) { phase in
+            if let image = phase.image {
+              image
+                .resizable()
+                .aspectRatio(contentMode: .fill)
+                .frame(width: 20, height: 20, alignment: .center)
+            } else {
+              Color.appButtonBackground
+                .frame(width: 20, height: 20, alignment: .center)
+            }
+          }
+          Text(viewModel.item.site)
+            .font(Font.system(size: 14))
+            .frame(maxWidth: .infinity, alignment: .topLeading)
+        }
+        .padding(.top, 10)
+        Text(viewModel.item.title)
+          // .font(.body)
+          // .fontWeight(.semibold)
+          .font(Font.system(size: 18, weight: .semibold))
+          .frame(maxWidth: .infinity, alignment: .topLeading)
+
+        Text(viewModel.item.author)
+          .font(Font.system(size: 14))
+          .foregroundColor(Color(hex: "898989"))
+          .frame(maxWidth: .infinity, alignment: .topLeading)
+
+        Color(hex: "2A2A2A")
+          .frame(height: 1)
+          .frame(maxWidth: .infinity, alignment: .center)
+          .padding(.vertical, 20)
+
+        if viewModel.isLoading {
+          ProgressView()
+            .task {
+              await viewModel.loadResult()
+            }
+            .frame(maxWidth: .infinity, maxHeight: .infinity)
+        } else {
+          Text(viewModel.item.summaryText)
+            .font(Font.system(size: 16))
+            // .font(.body)
+            .lineSpacing(12.0)
+            .frame(maxWidth: .infinity, alignment: .topLeading)
+          HStack {
+            Button(action: {}, label: {
+              HStack(alignment: .center) {
+                Text("Start listening")
+                  .font(Font.system(size: 14))
+                  .frame(height: 42, alignment: .center)
+                Image(systemName: "play.fill")
+                  .resizable()
+                  .frame(width: 10, height: 10)
+              }
+              .padding(.horizontal, 15)
+              .background(Color.blue)
+              .foregroundColor(.white)
+              .cornerRadius(18)
+            })
+            Spacer()
+          }
+          .padding(.top, 20)
+        }
+        Spacer()
+        if viewModel.showSwipeHint {
+          VStack {
+            Image.doubleChevronUp
+            Text("Swipe up for next article")
+              .foregroundColor(Color(hex: "898989"))
+          }
+          .padding(.bottom, 50)
+        }
+      }.frame(maxWidth: .infinity, maxHeight: .infinity)
+        .padding(.top, 100)
+        .padding(.horizontal, 15)
+
+    }
+}
+
+struct RatingView: View {
+    @State private var rating: Int = 0
+
+    var body: some View {
+      VStack(spacing: 30) {
+        Text("Rate today's digest")
+          .font(.title)
+          .padding(.vertical, 40)
+        Text("I liked the stories picked for today's digest")
+        RatingWidget()
+
+        Text("The stories were interesting")
+        RatingWidget()
+
+        Text("The voices sounded good")
+        RatingWidget()
+
+        Text("I liked the music")
+        RatingWidget()
+        Spacer()
+      }.padding(.top, 60)
+    }
+}
+
+struct StarView: View {
+    var isFilled: Bool
+    var body: some View {
+        Image(systemName: isFilled ? "star.fill" : "star")
+            .foregroundColor(isFilled ? Color.yellow : Color.gray)
+    }
+}
+
+struct RatingWidget: View {
+    @State private var rating: Int = 0
+    var body: some View {
+        HStack {
+            ForEach(1...5, id: \.self) { index in
+                StarView(isFilled: index <= rating)
+                    .onTapGesture {
+                        rating = index
+                    }
+            }
+        }
+        .padding()
+        .background(Color(hex: "313131"))
+        .cornerRadius(8)
+        // .shadow(radius: 3)
+    }
+}

apple/OmnivoreKit/Sources/App/Views/AudioPlayer/ExpandedAudioPlayer.swift

@@ -45,7 +45,7 @@
       if audioController.playbackError {
         return AnyView(Color.clear)
       }
-      if let itemID = audioController.itemAudioProperties?.itemID, audioController.isLoadingItem(itemID: itemID) {
+      if audioController.isLoadingItem(audioController.itemAudioProperties) {
         return AnyView(ProgressView())
       } else {
         return AnyView(Button(
@@ -98,11 +98,11 @@
           }
         ).padding(.trailing, 5)
 
-        if !(audioController.itemAudioProperties?.isArchived ?? false) {
+        if !((audioController.itemAudioProperties as? LinkedItemAudioProperties)?.isArchived ?? false) {
           Button(
             action: { performArchive() },
             label: {
-              if audioController.itemAudioProperties?.isArchived ?? false {
+              if (audioController.itemAudioProperties as? LinkedItemAudioProperties)?.isArchived ?? false {
                 Image
                   .toolbarUnarchive
                   .foregroundColor(Color.toolbarItemForeground)
@@ -130,20 +130,20 @@
       }
     }
 
-    func performViewArticle() {
-      if let objectID = audioController.itemAudioProperties?.objectID {
-        viewArticle(objectID)
-      }
-    }
+//    func performViewArticle() {
+//      if let objectID = audioController.itemAudioProperties?.objectID {
+//        viewArticle(objectID)
+//      }
+//    }
 
     func performDelete() {
-      if let objectID = audioController.itemAudioProperties?.objectID {
+      if let objectID = (audioController.itemAudioProperties as? LinkedItemAudioProperties)?.objectID {
         delete(objectID)
       }
     }
 
     func performArchive() {
-      if let objectID = audioController.itemAudioProperties?.objectID {
+      if let objectID = (audioController.itemAudioProperties as? LinkedItemAudioProperties)?.objectID {
         archive(objectID)
       }
     }
@@ -467,7 +467,7 @@
     }
 
     public var innerBody: some View {
-      if let itemAudioProperties = self.audioController.itemAudioProperties {
+      if let itemAudioProperties = self.audioController.itemAudioProperties as? LinkedItemAudioProperties {
         return AnyView(
           playerContent(itemAudioProperties)
             .tint(.appGrayTextContrast)

apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayer.swift

@@ -23,14 +23,14 @@
     public var body: some View {
       ZStack(alignment: .center) {
         presentingView
-        if let itemAudioProperties = self.audioController.itemAudioProperties {
+        if self.audioController.itemAudioProperties != nil {
           ZStack(alignment: .bottom) {
             Color.systemBackground.edgesIgnoringSafeArea(.bottom)
               .frame(height: expanded ? 0 : 110, alignment: .bottom)
 
             VStack {
               Spacer(minLength: 0)
-              MiniPlayerViewer(itemAudioProperties: itemAudioProperties)
+              MiniPlayerViewer()
             }
           }
         }

apple/OmnivoreKit/Sources/App/Views/AudioPlayer/MiniPlayerViewer.swift

@@ -13,8 +13,6 @@
 
     @State var expanded = true
 
-    let itemAudioProperties: LinkedItemAudioProperties
-
     var playPauseButtonImage: String {
       switch audioController.state {
       case .playing:
@@ -32,7 +30,7 @@
       if audioController.playbackError {
         return AnyView(Color.clear)
       }
-      if let itemID = audioController.itemAudioProperties?.itemID, audioController.isLoadingItem(itemID: itemID) {
+      if audioController.isLoadingItem(audioController.itemAudioProperties) {
         return AnyView(ProgressView())
       } else {
         return AnyView(Button(
@@ -84,8 +82,8 @@
       .buttonStyle(PlainButtonStyle())
     }
 
-    func artwork(_ itemAudioProperties: LinkedItemAudioProperties, forDimensions dim: Double) -> some View {
-      if let imageURL = itemAudioProperties.imageURL {
+    func artwork(_ itemAudioProperties: AudioItemProperties?, forDimensions dim: Double) -> some View {
+      if let imageURL = itemAudioProperties?.imageURL {
         return AnyView(AsyncImage(url: imageURL) { phase in
           if let image = phase.image {
             image
@@ -125,9 +123,9 @@
             Text("There was an error playing back your audio.").foregroundColor(Color.red).font(.footnote)
             Spacer(minLength: 0)
           } else {
-            artwork(itemAudioProperties, forDimensions: 50)
+            artwork(audioController.itemAudioProperties, forDimensions: 50)
 
-            Text(itemAudioProperties.title)
+            Text(audioController.itemAudioProperties?.title ?? "")
               .font(Font.system(size: 17, weight: .medium))
               .fixedSize(horizontal: false, vertical: true)
               .lineLimit(2)

apple/OmnivoreKit/Sources/App/Views/BottomBars/CustomTabBar.swift

@@ -2,16 +2,31 @@ import Foundation
 import SwiftUI
 
 struct CustomTabBar: View {
+  let displayTabs: [String]
   @Binding var selectedTab: String
-  let hideFollowingTab: Bool
 
   var body: some View {
     HStack(spacing: 0) {
-      if !hideFollowingTab {
-        TabBarButton(key: "following", image: Image.tabFollowing, selectedTab: $selectedTab)
+      if displayTabs.contains("following") {
+        TabBarButton(key: "following",
+                     image: Image.tabFollowing,
+                     selectedTab: $selectedTab,
+                     selectionColor: Color(hex: "EE8232"))
+      }
+      if displayTabs.contains("digest") {
+        TabBarButton(key: "digest",
+                     image: Image.tabDigest,
+                     selectedTab: $selectedTab,
+                     selectedImage: Image.tabDigestSelected)
+      }
+      if displayTabs.contains("inbox") {
+        TabBarButton(key: "inbox",
+                     image: Image.tabLibrary,
+                     selectedTab: $selectedTab)
+      }
+      if displayTabs.contains("profile") {
+        TabBarButton(key: "profile", image: Image.tabProfile, selectedTab: $selectedTab)
       }
-      TabBarButton(key: "inbox", image: Image.tabLibrary, selectedTab: $selectedTab)
-      TabBarButton(key: "profile", image: Image.tabProfile, selectedTab: $selectedTab)
     }
     .padding(.top, 10)
     .padding(.bottom, 10)
@@ -23,6 +38,8 @@ struct TabBarButton: View {
   let key: String
   let image: Image
   @Binding var selectedTab: String
+  var selectedImage: Image?
+  var selectionColor: Color?
 
   var body: some View {
     Button(action: {
@@ -31,13 +48,20 @@ struct TabBarButton: View {
       }
       selectedTab = key
     }, label: {
-      image
-        .resizable()
-        .renderingMode(.template)
-        .aspectRatio(contentMode: .fit)
+     tabImage
         .frame(width: 28, height: 28)
-        .foregroundColor(selectedTab == key ? Color.blue : Color.themeTabButtonColor)
         .frame(maxWidth: .infinity)
     }).buttonStyle(.plain)
   }
+  
+  var tabImage: some View {
+    if let selectedImage = selectedImage, selectedTab == key {
+      return AnyView(selectedImage
+        .resizable()
+        .aspectRatio(contentMode: .fit))
+    } else {
+      return AnyView(image
+        .foregroundColor(selectedTab == key ? selectionColor ?? Color.blue  : Color.themeTabButtonColor))
+    }
+  }
 }

apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift

@@ -191,6 +191,7 @@ struct AnimatingCellHeight: AnimatableModifier {
     @State var isListScrolled = false
     @State var listTitle = ""
     @State var showExpandedAudioPlayer = false
+    @State var showLibraryDigest = false
 
     @Binding var isEditMode: EditMode
 
@@ -203,6 +204,9 @@ struct AnimatingCellHeight: AnimatableModifier {
     @ObservedObject var viewModel: HomeFeedViewModel
     @State private var selection = Set<String>()
 
+    @AppStorage("LibraryList::digestEnabled") var digestEnabled = false
+    @AppStorage("LibraryList::hasCheckedForDigestFeature") var hasCheckedForDigestFeature = false
+
     init(viewModel: HomeFeedViewModel, isEditMode: Binding<EditMode>) {
       _viewModel = ObservedObject(wrappedValue: viewModel)
       _isEditMode = isEditMode
@@ -224,7 +228,7 @@ struct AnimatingCellHeight: AnimatableModifier {
     }
 
     var body: some View {
-      ZStack {        
+      ZStack {
         HomeFeedView(
           listTitle: $listTitle,
           isListScrolled: $isListScrolled,
@@ -264,8 +268,8 @@ struct AnimatingCellHeight: AnimatableModifier {
           VStack(spacing: 0) {
             Spacer()
 
-            if let audioProperties = audioController.itemAudioProperties {
-              MiniPlayerViewer(itemAudioProperties: audioProperties)
+            if audioController.itemAudioProperties != nil {
+              MiniPlayerViewer()
                 .padding(.top, 10)
                 .padding(.bottom, 20)
                 .background(Color.themeTabBarColor)
@@ -316,6 +320,15 @@ struct AnimatingCellHeight: AnimatableModifier {
           }
         )
       }
+      .fullScreenCover(isPresented: $showLibraryDigest) {
+        if #available(iOS 17.0, *) {
+          NavigationView {
+            FullScreenDigestView(dataService: dataService, audioController: audioController)
+          }
+        } else {
+          Text("Sorry digest is only available on iOS 17 and above")
+        }
+      }
       .toolbar {
         toolbarItems
       }
@@ -338,6 +351,20 @@ struct AnimatingCellHeight: AnimatableModifier {
           viewModel.stopUsingFollowingPrimer = true
         }
       }
+      .task {
+        do {
+          if let viewer = try await dataService.fetchViewer() {
+            digestEnabled = viewer.digestEnabled ?? false
+            if !hasCheckedForDigestFeature {
+              hasCheckedForDigestFeature = true
+              // selectedTab = "digest"
+            }
+          }
+        } catch {
+          print("ERROR FETCHING VIEWER: ", error)
+          print("")
+        }
+      }
       .environment(\.editMode, self.$isEditMode)
       .navigationBarTitleDisplayMode(.inline)
     }
@@ -382,6 +409,14 @@ struct AnimatingCellHeight: AnimatableModifier {
               if isEditMode == .active {
                 Button(action: { isEditMode = .inactive }, label: { Text("Cancel") })
               } else {
+                if #available(iOS 17.0, *) {
+                  Button(
+                    action: { showLibraryDigest = true },
+                    label: { Image.tabDigestSelected }
+                  )
+                  .buttonStyle(.plain)
+                  .padding(.trailing, 4)
+                }
                 if prefersListLayout {
                   Button(
                     action: { isEditMode = isEditMode == .active ? .inactive : .active },
@@ -471,12 +506,12 @@ struct AnimatingCellHeight: AnimatableModifier {
     let showFeatureCards: Bool
     var slideTransition: PresentationLinkTransition {
       PresentationLinkTransition.slide(
-        options: PresentationLinkTransition.SlideTransitionOptions(edge: .trailing,
-                                                                   options:
-                                                                    PresentationLinkTransition.Options(
-                                                                      modalPresentationCapturesStatusBarAppearance: true
-                                                                    )
-                                                                  ))
+        options: PresentationLinkTransition.SlideTransitionOptions(
+          edge: .trailing,
+          options: PresentationLinkTransition.Options(
+            modalPresentationCapturesStatusBarAppearance: true
+          )
+        ))
     }
 
     var body: some View {
@@ -484,12 +519,12 @@ struct AnimatingCellHeight: AnimatableModifier {
         if let linkRequest = viewModel.linkRequest, viewModel.currentListConfig?.hasReadNowSection ?? false {
           PresentationLink(
             transition: PresentationLinkTransition.slide(
-              options: PresentationLinkTransition.SlideTransitionOptions(edge: .trailing,
-                                                                         options:
-                                                                         PresentationLinkTransition.Options(
-                                                                           modalPresentationCapturesStatusBarAppearance: true,
-                                                                           preferredPresentationBackgroundColor: ThemeManager.currentBgColor
-                                                                         ))),
+              options: PresentationLinkTransition.SlideTransitionOptions(
+                edge: .trailing,
+                options: PresentationLinkTransition.Options(
+                  modalPresentationCapturesStatusBarAppearance: true,
+                  preferredPresentationBackgroundColor: ThemeManager.currentBgColor
+                ))),
             isPresented: $viewModel.presentWebContainer,
             destination: {
               WebReaderLoadingContainer(requestID: linkRequest.serverID)

apple/OmnivoreKit/Sources/App/Views/LibraryTabView.swift

@@ -1,10 +1,3 @@
-//
-//  File.swift
-//
-//
-//  Created by Jackson Harper on 6/29/23.
-//
-
 import Foundation
 import Models
 import Services
@@ -76,6 +69,30 @@ struct LibraryTabView: View {
   @State var operationStatus: OperationStatus = .none
   @State var operationMessage: String?
 
+  @State var digestEnabled = false
+
+  var showDigest: Bool {
+    if digestEnabled, #available(iOS 17.0, *) {
+      return true
+    }
+    return false
+  }
+
+  var displayTabs: [String] {
+    var res = [String]()
+    if !hideFollowingTab {
+      res.append("following")
+    }
+    if showDigest {
+      res.append("digest")
+    }
+    res.append("inbox")
+    if !showDigest {
+      res.append("profile")
+    }
+    return res
+  }
+
   var body: some View {
     VStack(spacing: 0) {
       WindowLink(level: .alert, transition: .move(edge: .bottom), isPresented: $showOperationToast) {
@@ -116,19 +133,32 @@ struct LibraryTabView: View {
           }.tag("following")
         }
 
-        NavigationView {
-          HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode)
-            .navigationBarTitleDisplayMode(.inline)
-            .navigationViewStyle(.stack)
-        }.tag("inbox")
+        if showDigest, #available(iOS 17.0, *) {
+          NavigationView {
+            DigestView(dataService: dataService)
+              .navigationBarTitleDisplayMode(.inline)
+              .navigationViewStyle(.stack)
+          }.tag("digest")
+          NavigationView {
+            HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode)
+              .navigationBarTitleDisplayMode(.inline)
+              .navigationViewStyle(.stack)
+          }.tag("inbox")
+        } else {
+          NavigationView {
+            HomeFeedContainerView(viewModel: inboxViewModel, isEditMode: $isEditMode)
+              .navigationBarTitleDisplayMode(.inline)
+              .navigationViewStyle(.stack)
+          }.tag("inbox")
+          NavigationView {
+            ProfileView()
+              .navigationViewStyle(.stack)
+          }.tag("profile")
+        }
 
-        NavigationView {
-          ProfileView()
-            .navigationViewStyle(.stack)
-        }.tag("profile")
       }
-      if let audioProperties = audioController.itemAudioProperties {
-        MiniPlayerViewer(itemAudioProperties: audioProperties)
+      if audioController.itemAudioProperties != nil {
+        MiniPlayerViewer()
           .onTapGesture {
             showExpandedAudioPlayer = true
           }
@@ -138,7 +168,9 @@ struct LibraryTabView: View {
           .frame(maxWidth: .infinity)
       }
       if isEditMode != .active {
-        CustomTabBar(selectedTab: $selectedTab, hideFollowingTab: hideFollowingTab)
+        CustomTabBar(
+          displayTabs: displayTabs,
+          selectedTab: $selectedTab)
           .padding(0)
       }
     }

apple/OmnivoreKit/Sources/App/Views/PrimaryContentView.swift

@@ -52,7 +52,7 @@ import Transmission
       return AnyView(splitView)
     #endif
   }
-  
+
   func startTimer(amount: Int) {
     self.snackbarTimer = Timer.scheduledTimer(withTimeInterval: TimeInterval(amount / 1000), repeats: false) { _ in
       DispatchQueue.main.async {

apple/OmnivoreKit/Sources/App/Views/WebReader/WebReader.swift

@@ -11,6 +11,7 @@ struct WebReader: PlatformViewRepresentable {
   let articleContent: ArticleContent
   let openLinkAction: (URL) -> Void
   let tapHandler: () -> Void
+  let explainHandler: ((String) -> Void)?
   let scrollPercentHandler: (Int) -> Void
   let webViewActionHandler: (WKScriptMessage, WKScriptMessageReplyHandler?) -> Void
   let navBarVisibilityUpdater: (Bool) -> Void
@@ -51,6 +52,7 @@ struct WebReader: PlatformViewRepresentable {
     let contentController = WKUserContentController()
 
     webView.tapHandler = tapHandler
+    webView.explainHandler = explainHandler
     webView.navigationDelegate = context.coordinator
     webView.configuration.userContentController = contentController
     webView.configuration.userContentController.removeAllScriptMessageHandlers()

apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift

@@ -41,6 +41,8 @@ struct WebReaderContainerView: View {
   @State var displayLinkSheet = false
   @State var linkToOpen: URL?
 
+  @State var showExplainSheet = false
+
   @EnvironmentObject var dataService: DataService
   @EnvironmentObject var audioController: AudioController
   @Environment(\.openURL) var openURL
@@ -92,6 +94,11 @@ struct WebReaderContainerView: View {
     }
   }
 
+  private func explainHandler(text: String) {
+    viewModel.explainText = String(text)
+    showExplainSheet = true
+  }
+
   private func handleHighlightAction(message: WKScriptMessage) {
     guard let messageBody = message.body as? [String: String] else { return }
     guard let actionID = messageBody["actionID"] else { return }
@@ -126,7 +133,7 @@ struct WebReaderContainerView: View {
 
   #if os(iOS)
     var audioNavbarItem: some View {
-      if !audioController.playbackError, audioController.isLoadingItem(itemID: item.unwrappedID) {
+      if audioController.isLoadingItem(audioController.itemAudioProperties) {
         return AnyView(ProgressView()
           .padding(.horizontal))
       } else {
@@ -271,6 +278,13 @@ struct WebReaderContainerView: View {
         Spacer()
       #endif
 
+//      Button(
+//        action: { showExplainSheet = true },
+//        label: { Image(systemName: "sparkles") }
+//      )
+//      .buttonStyle(.plain)
+//      .padding(.trailing, 4)
+
       Button(
         action: { showLabelsModal = true },
         label: {
@@ -378,6 +392,7 @@ struct WebReaderContainerView: View {
             #endif
           },
           tapHandler: tapHandler,
+          explainHandler: explainHandler,
           scrollPercentHandler: scrollPercentHandler,
           webViewActionHandler: webViewActionHandler,
           navBarVisibilityUpdater: { visible in
@@ -600,8 +615,8 @@ struct WebReaderContainerView: View {
             .offset(y: navBarVisible ? 0 : -150)
 
           Spacer()
-          if let audioProperties = audioController.itemAudioProperties {
-            MiniPlayerViewer(itemAudioProperties: audioProperties)
+          if audioController.itemAudioProperties != nil {
+            MiniPlayerViewer()
               .padding(.top, 10)
               .padding(.bottom, showBottomBar ? 10 : 40)
               .background(Color.themeTabBarColor)
@@ -633,7 +648,6 @@ struct WebReaderContainerView: View {
       try? WebViewManager.shared().dispatchEvent(.saveReadPosition)
     }
     .onDisappear {
-      // WebViewManager.shared().loadHTMLString("<html></html>", baseURL: nil)
       WebViewManager.shared().loadHTMLString(WebReaderContent.emptyContent(isDark: Color.isDarkMode), baseURL: nil)
     }
     .onReceive(NotificationCenter.default.publisher(for: Notification.Name("PopToRoot"))) { _ in
@@ -674,7 +688,7 @@ struct WebReaderContainerView: View {
     shareActionID = UUID()
   }
   
-  func print() {
+  func printReader() {
     shareActionID = UUID()
   }
 

apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderViewModel.swift

@@ -21,6 +21,8 @@ struct SafariWebLink: Identifiable {
   @Published var showOperationToast: Bool = false
   @Published var operationStatus: OperationStatus = .none
 
+  @Published var explainText: String?
+
   func hasOriginalUrl(_ item: Models.LibraryItem) -> Bool {
     if let pageURLString = item.pageURLString, let host = URL(string: pageURLString)?.host {
       if host == "omnivore.app" {

apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22522" systemVersion="23B81" minimumToolsVersion="Automatic" sourceLanguage="Swift" userDefinedModelVersionIdentifier="">
+<model type="com.apple.IDECoreDataModeler.DataModel" documentVersion="1.0" lastSavedToolsVersion="22757" systemVersion="23B81" minimumToolsVersion="Automatic" sourceLanguage="Swift" userDefinedModelVersionIdentifier="">
     <entity name="Filter" representedClassName="Filter" syncable="YES" codeGenerationType="class">
         <attribute name="defaultFilter" optional="YES" attributeType="Boolean" usesScalarValueType="YES"/>
         <attribute name="filter" optional="YES" attributeType="String"/>
@@ -137,6 +137,7 @@
         <attribute name="username" optional="YES" attributeType="String"/>
     </entity>
     <entity name="Viewer" representedClassName="Viewer" syncable="YES" codeGenerationType="class">
+        <attribute name="digestEnabled" optional="YES" attributeType="Boolean" usesScalarValueType="YES"/>
         <attribute name="name" attributeType="String"/>
         <attribute name="profileImageURL" optional="YES" attributeType="String"/>
         <attribute name="userID" attributeType="String"/>

apple/OmnivoreKit/Sources/Models/DataModels/Feature.swift

@@ -0,0 +1,10 @@
+
+public struct FeatureInternal {
+  public let name: String
+  public let enabled: Bool
+
+  public init(name: String, enabled: Bool) {
+    self.name = name
+    self.enabled = enabled
+  }
+}

apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift

@@ -27,8 +27,7 @@ public struct LinkedItemSyncResult {
               hasMore: Bool,
               mostRecentUpdatedAt: Date?,
               oldestUpdatedAt: Date?,
-              isEmpty: Bool)
-  {
+              isEmpty: Bool) {
     self.updatedItemIDs = updatedItemIDs
     self.cursor = cursor
     self.hasMore = hasMore
@@ -38,7 +37,41 @@ public struct LinkedItemSyncResult {
   }
 }
 
-public struct LinkedItemAudioProperties {
+public enum AudioItemType {
+  case digest
+  case libraryItem
+}
+
+public protocol AudioItemProperties {
+  var audioItemType: AudioItemType {
+    get
+  }
+  var itemID: String {
+    get
+  }
+  var title: String {
+    get
+  }
+  var byline: String? {
+    get
+  }
+  var imageURL: URL? {
+    get
+  }
+  var language: String? {
+    get
+  }
+  var startIndex: Int {
+    get
+  }
+  var startOffset: Double {
+    get
+  }
+}
+
+public struct LinkedItemAudioProperties: AudioItemProperties {
+  public let audioItemType = AudioItemType.libraryItem
+
   public let itemID: String
   public let objectID: NSManagedObjectID
   public let title: String

apple/OmnivoreKit/Sources/Services/AudioSession/AudioController.swift

@@ -33,7 +33,7 @@
     @Published public var currentAudioIndex: Int = 0
     @Published public var readText: String = ""
     @Published public var unreadText: String = ""
-    @Published public var itemAudioProperties: LinkedItemAudioProperties?
+    @Published public var itemAudioProperties: AudioItemProperties?
 
     @Published public var timeElapsed: TimeInterval = 0
     @Published public var duration: TimeInterval = 0
@@ -72,7 +72,7 @@
       observer = nil
     }
 
-    public func play(itemAudioProperties: LinkedItemAudioProperties) {
+    public func play(itemAudioProperties: AudioItemProperties) {
       stop()
 
       playbackError = false
@@ -539,11 +539,14 @@
       state == .playing
     }
 
-    public func isLoadingItem(itemID: String) -> Bool {
+    public func isLoadingItem(_ audioItem: AudioItemProperties?) -> Bool {
       if state == .reachedEnd {
         return false
       }
-      return itemAudioProperties?.itemID == itemID && isLoading
+      if audioItem?.itemID == nil {
+        return false
+      }
+      return itemAudioProperties?.itemID == audioItem?.itemID && isLoading
     }
 
     public func isPlayingItem(itemID: String) -> Bool {
@@ -860,8 +863,19 @@
       }
       return ""
     }
-
+    
     func downloadSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? {
+      switch(self.itemAudioProperties?.audioItemType) {
+      case .digest:
+        return try await downloadDigestItemSpeechFile(itemID: itemID, priority: priority)
+      case .libraryItem:
+        return try await downloadLibraryItemSpeechFile(itemID: itemID, priority: priority)
+      case .none:
+        return nil
+      }
+    }
+
+    func downloadLibraryItemSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? {
       let decoder = JSONDecoder()
       let speechFileUrl = pathForSpeechFile(itemID: itemID)
 
@@ -910,6 +924,60 @@
       return nil
     }
 
+    func downloadDigestItemSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? {
+      let decoder = JSONDecoder()
+      let speechFileUrl = URL.om_documentsDirectory.appendingPathComponent("digest").appendingPathComponent("speech-\(currentVoice).json")
+
+      if FileManager.default.fileExists(atPath: speechFileUrl.path) {
+        let data = try Data(contentsOf: speechFileUrl)
+        document = try decoder.decode(SpeechDocument.self, from: data)
+        // If we can't load it from disk we make the API call
+        if let document = document {
+          return document
+        }
+      }
+
+      let path = "/api/digest/v1/"
+      guard let url = URL(string: path, relativeTo: dataService.appEnvironment.serverBaseURL) else {
+        throw BasicError.message(messageText: "Invalid audio URL")
+      }
+
+      var request = URLRequest(url: url)
+      request.httpMethod = "GET"
+      for (header, value) in dataService.networker.defaultHeaders {
+        request.setValue(value, forHTTPHeaderField: header)
+      }
+
+      let result: (Data, URLResponse)? = try? await URLSession.shared.data(for: request)
+      guard let httpResponse = result?.1 as? HTTPURLResponse, 200 ..< 300 ~= httpResponse.statusCode else {
+        throw BasicError.message(messageText: "audioFetch failed. no response or bad status code.")
+      }
+
+      guard let data = result?.0 else {
+        throw BasicError.message(messageText: "audioFetch failed. no data received.")
+      }
+
+      let str = String(decoding: data, as: UTF8.self)
+      print("result digest file: ", str)
+
+      do {
+        let digest = try JSONDecoder().decode(DigestResult.self, from: data)
+        let directory = URL.om_documentsDirectory.appendingPathComponent("digest")
+         // do {
+            try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
+            try data.write(to: speechFileUrl)
+            return digest.speechFile
+//          } catch {
+//            print("error writing file", error)
+//          }
+        // }
+      } catch {
+        print("error with digest file", error)
+      }
+
+      return nil
+    }
+
     func getSpeechFile(itemID: String, priority: DownloadPriority) async throws -> SpeechDocument? {
       document = try await downloadSpeechFile(itemID: itemID, priority: priority)
       return document

apple/OmnivoreKit/Sources/Services/AudioSession/SpeechSynthesizer.swift

@@ -20,7 +20,7 @@ struct UtteranceRequest: Codable {
   let isOpenAIVoice: Bool
 }
 
-struct Utterance: Decodable {
+public struct Utterance: Decodable {
   public let idx: String
   public let text: String
   public let voice: String?
@@ -39,7 +39,7 @@ struct Utterance: Decodable {
   }
 }
 
-struct SpeechDocument: Decodable {
+public struct SpeechDocument: Decodable {
   static let averageWPM: Double = 195
 
   public let pageId: String

apple/OmnivoreKit/Sources/Services/Authentication/AuthModels.swift

@@ -75,7 +75,7 @@ extension LoginError {
       return .unauthorized
     case .unknown:
       return .unknown
-    case .pendingEmailVerification:
+    case .pendingEmailVerification, .stillProcessing:
       return .pendingEmailVerification
     }
   }

apple/OmnivoreKit/Sources/Services/DataService/AI/AITasks.swift

@@ -0,0 +1,118 @@
+import CoreData
+import Foundation
+import Models
+import Utils
+
+struct AITaskRequest: Decodable {
+  public let requestId: String
+}
+
+public struct DigestResult: Decodable {
+  public let id: String
+  public let title: String
+  public let content: String
+  public let urlsToAudio: [String]
+  public let speechFile: SpeechDocument
+
+  public let jobState: String
+}
+
+public struct DigestItem: Decodable {
+  public let id: String
+  public let site: String
+  public let siteIcon: URL?
+  public let author: String
+  public let title: String
+  public let summaryText: String
+  public let keyPointsText: String
+  public let highlightsText: String
+  public init(id: String, site: String, siteIcon: URL?, 
+              author: String, title: String, summaryText: String,
+              keyPointsText: String, highlightsText: String) {
+    self.id = id
+    self.site = site
+    self.siteIcon = siteIcon
+    self.author = author
+    self.title = title
+    self.summaryText = summaryText
+    self.keyPointsText = keyPointsText
+    self.highlightsText = highlightsText
+  }
+}
+
+extension DataService {
+//  public func createAITask(extraText: String?, libraryItemId: String, promptName: String) async throws -> String? {
+//    let jsonData = try JSONSerialization.data(withJSONObject: [
+//      "libraryItemId": libraryItemId,
+//      "promptName": promptName,
+//      "extraText": extraText
+//    ])
+//
+//    let urlRequest = URLRequest.create(
+//      baseURL: appEnvironment.serverBaseURL,
+//      urlPath: "/api/ai-task",
+//      requestMethod: .post(params: jsonData),
+//      includeAuthToken: true
+//    )
+//    let resource = ServerResource<AITaskRequest>(
+//      urlRequest: urlRequest,
+//      decode: AITaskRequest.decode
+//    )
+//
+//    do {
+//      let taskRequest = try await networker.urlSession.performRequest(resource: resource)
+//      return taskRequest.requestId
+//    } catch {
+//      return nil
+//    }
+//  }
+
+  // Function to poll the status of the AI task with timeout
+  public func getLatestDigest(timeoutInterval: TimeInterval) async throws -> DigestResult? {
+    var count = 0
+      let startTime = Date()
+      while true {
+        count += 1
+        if count > 3 {
+          return nil
+        }
+          do {
+              // Check if timeout has occurred
+              if -startTime.timeIntervalSinceNow >= timeoutInterval {
+                  throw NSError(domain: "Timeout Error", code: -1, userInfo: nil)
+              }
+
+              let urlRequest = URLRequest.create(
+                baseURL: appEnvironment.serverBaseURL,
+                urlPath: "/api/digest/v1/",
+                requestMethod: .get,
+                includeAuthToken: true
+              )
+
+            let resource = ServerResource<DigestResult>(
+              urlRequest: urlRequest,
+              decode: DigestResult.decode
+            )
+
+            do {
+              let digest = try await networker.urlSession.performRequest(resource: resource)
+              print("GOT RESPONSE: ", digest)
+              return digest
+            } catch {
+              print("ERROR FETCHING TASK: ", error)
+//              if let response = error as? ServerError {
+//                if response != .stillProcessing {
+//                  return nil
+//                }
+//              }
+            }
+              // Wait for some time before polling again
+              try? await Task.sleep(nanoseconds: 3_000_000_000)
+          } catch let error {
+              throw error
+          }
+      }
+  }
+}
+
+

apple/OmnivoreKit/Sources/Services/DataService/Errors/ServerError.swift

@@ -10,6 +10,7 @@ public enum ServerError: String, Error {
   case unauthenticated
   case timeout
   case unknown
+  case stillProcessing
   case pendingEmailVerification
 }
 
@@ -22,6 +23,8 @@ extension ServerError {
     case 401?, 403?:
       self = .unauthenticated
       return
+    case 202:
+      self = .stillProcessing
     default:
       break
     }

apple/OmnivoreKit/Sources/Services/DataService/Mutations/SetRuleMutation.swift

@@ -15,6 +15,8 @@ public enum RuleActionType {
   case delete
   case markAsRead
   case sendNotification
+  case export
+  case webhook
 
   static func from(_ other: Enums.RuleActionType) -> RuleActionType {
     switch other {
@@ -28,6 +30,10 @@ public enum RuleActionType {
       return .sendNotification
     case .delete:
       return .delete
+    case Enums.RuleActionType.export:
+      return .export
+    case Enums.RuleActionType.webhook:
+      return .webhook
     }
   }
 }

apple/OmnivoreKit/Sources/Services/DataService/Queries/ViewerFetcher.swift

@@ -17,7 +17,9 @@ public extension DataService {
         profileImageURL: try $0.profile(
           selection: .init { try $0.pictureUrl() }
         ),
-        intercomHash: try $0.intercomHash()
+        intercomHash: try $0.intercomHash(),
+        digestEnabled: true // (try $0.featureList(selection: featureSelection.list.nullable)?
+              // .filter { $0.enabled && $0.name == "digest" } ?? []).count > 0
       )
     }
 
@@ -65,6 +67,7 @@ public struct ViewerInternal {
   public let name: String
   public let profileImageURL: String?
   public let intercomHash: String?
+  public let digestEnabled: Bool?
 
   func persist(context: NSManagedObjectContext) throws {
     try context.performAndWait {
@@ -73,6 +76,7 @@ public struct ViewerInternal {
       viewer.username = username
       viewer.name = name
       viewer.profileImageURL = profileImageURL
+      viewer.digestEnabled = digestEnabled ?? false
 
       do {
         try context.save()

apple/OmnivoreKit/Sources/Services/DataService/Selections/FeatureSelection.swift

@@ -0,0 +1,6 @@
+import Models
+import SwiftGraphQL
+
+let featureSelection = Selection.Feature {
+  FeatureInternal(name: try $0.name(), enabled: try $0.grantedAt() != nil)
+}

apple/OmnivoreKit/Sources/Views/Article/OmnivoreWebView.swift

@@ -23,6 +23,7 @@ public final class OmnivoreWebView: WKWebView {
   #endif
 
   public var tapHandler: (() -> Void)?
+  public var explainHandler: ((String) -> Void)?
 
   private var currentMenu: ContextMenu = .defaultMenu
 
@@ -298,6 +299,7 @@ public final class OmnivoreWebView: WKWebView {
       case #selector(removeSelection): return true
       case #selector(copy(_:)): return true
       case #selector(setLabels(_:)): return true
+      case #selector(explainSelection): return true
 
       case Selector(("_lookup:")): return (currentMenu == .defaultMenu)
       case Selector(("_define:")): return (currentMenu == .defaultMenu)
@@ -334,6 +336,18 @@ public final class OmnivoreWebView: WKWebView {
       hideMenu()
     }
 
+    @objc private func explainSelection() {
+      Task {
+        let selection = try? await self.evaluateJavaScript("window.getSelection().toString()")
+        if let selection = selection as? String, let explainHandler = explainHandler {
+          print("Explaining \(selection)")
+          explainHandler(selection)
+        } else {
+          showInReaderSnackbar("Error getting text to explain")
+        }
+      }
+    }
+
     @objc private func shareSelection() {
       do {
         try dispatchEvent(.share)
@@ -386,7 +400,8 @@ public final class OmnivoreWebView: WKWebView {
             return
           }
           let highlight = UICommand(title: LocalText.genericHighlight, action: #selector(highlightSelection))
-          items = [highlight, annotate]
+          // let explain = UICommand(title: "Explain", action: #selector(explainSelection))
+          items = [highlight, /* explain, */ annotate]
         } else {
           let remove = UICommand(title: "Remove", action: #selector(removeSelection))
           let setLabels = UICommand(title: LocalText.labelsGeneric, action: #selector(setLabels))

apple/OmnivoreKit/Sources/Views/Images/Images.swift

@@ -9,6 +9,9 @@ public extension Image {
 
   static var tabFollowing: Image { Image("_tab_following", bundle: .module).renderingMode(.template) }
   static var tabLibrary: Image { Image("_tab_library", bundle: .module).renderingMode(.template) }
+  static var tabDigest: Image { Image("_tab_digest", bundle: .module).renderingMode(.template) }
+  static var tabDigestSelected: Image { Image("_tab_digest_selected", bundle: .module) }
+
   static var tabSearch: Image { Image("_tab_search", bundle: .module).renderingMode(.template) }
   static var tabHighlights: Image { Image("_tab_highlights", bundle: .module).renderingMode(.template) }
   static var tabProfile: Image { Image("_tab_profile", bundle: .module).renderingMode(.template) }
@@ -49,4 +52,7 @@ public extension Image {
   static var flairNewsletter: Image { Image("flair-newsletter", bundle: .module) }
   static var flairPinned: Image { Image("flair-pinned", bundle: .module) }
   static var flairRecommended: Image { Image("flair-recommended", bundle: .module) }
+  
+  static var doubleChevronUp: Image { Image("double_chevron_up", bundle: .module) }
+
 }

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/Contents.json

@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "filename" : "_tab_digest.svg",
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest.imageset/_tab_digest.svg

@@ -0,0 +1,18 @@
+<svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_10231_7167)">
+<path d="M12.5 3.5V5.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.5 19.5V21.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.5 8.5V16.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.5 16.5V19.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M4.5 10.5V15.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M20.5 10.5V16.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.5 6.5V13.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.5 9.5V6.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.5 19.5V12.5" stroke="#8E8E93" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_10231_7167">
+<rect width="24" height="24" fill="white" transform="translate(0.5 0.5)"/>
+</clipPath>
+</defs>
+</svg>

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/Contents.json

@@ -0,0 +1,21 @@
+{
+  "images" : [
+    {
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "filename" : "_tab_digest_selected.svg",
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/_tab_digest_selected.imageset/_tab_digest_selected.svg

@@ -0,0 +1,18 @@
+<svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_10210_6950)">
+<path d="M12.5 3.5V5.5" stroke="#767AF8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.5 19.5V21.5" stroke="#DE76F8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M12.5 8.5V16.5" stroke="#767AF8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.5 16.5V19.5" stroke="#76AAF8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M4.5 10.5V15.5" stroke="#76F8D1" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M20.5 10.5V16.5" stroke="#76F8D1" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M8.5 6.5V13.5" stroke="#CF76F8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.5 9.5V6.5" stroke="#CF76F8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.5 19.5V12.5" stroke="#76AAF8" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_10210_6950">
+<rect width="24" height="24" fill="white" transform="translate(0.5 0.5)"/>
+</clipPath>
+</defs>
+</svg>

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/Contents.json

@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "filename" : "digest-archive-button.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-archive-button.imageset/digest-archive-button.svg

@@ -0,0 +1,13 @@
+<svg width="35" height="35" viewBox="0 0 35 35" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="34" height="34" rx="17" stroke="#3D3D3D"/>
+<g clip-path="url(#clip0_9916_1773)">
+<path d="M10 12.5002C10 12.0581 10.1756 11.6342 10.4882 11.3217C10.8007 11.0091 11.2246 10.8335 11.6667 10.8335H23.3333C23.7754 10.8335 24.1993 11.0091 24.5118 11.3217C24.8244 11.6342 25 12.0581 25 12.5002C25 12.9422 24.8244 13.3661 24.5118 13.6787C24.1993 13.9912 23.7754 14.1668 23.3333 14.1668H11.6667C11.2246 14.1668 10.8007 13.9912 10.4882 13.6787C10.1756 13.3661 10 12.9422 10 12.5002Z" stroke="#EDEDED" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M11.6667 14.1665V22.4998C11.6667 22.9419 11.8423 23.3658 12.1549 23.6783C12.4675 23.9909 12.8914 24.1665 13.3334 24.1665H21.6667C22.1088 24.1665 22.5327 23.9909 22.8453 23.6783C23.1578 23.3658 23.3334 22.9419 23.3334 22.4998V14.1665" stroke="#EDEDED" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M15.8333 17.5H19.1666" stroke="#EDEDED" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_9916_1773">
+<rect width="20" height="20" fill="white" transform="translate(7.5 7.5)"/>
+</clipPath>
+</defs>
+</svg>

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/Contents.json

@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "filename" : "digest-dots-button.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-dots-button.imageset/digest-dots-button.svg

@@ -0,0 +1,13 @@
+<svg width="35" height="35" viewBox="0 0 35 35" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="34" height="34" rx="17" stroke="#3D3D3D"/>
+<g clip-path="url(#clip0_9916_1805)">
+<path d="M10.8333 17.4998C10.8333 17.7209 10.921 17.9328 11.0773 18.0891C11.2336 18.2454 11.4456 18.3332 11.6666 18.3332C11.8876 18.3332 12.0996 18.2454 12.2558 18.0891C12.4121 17.9328 12.4999 17.7209 12.4999 17.4998C12.4999 17.2788 12.4121 17.0669 12.2558 16.9106C12.0996 16.7543 11.8876 16.6665 11.6666 16.6665C11.4456 16.6665 11.2336 16.7543 11.0773 16.9106C10.921 17.0669 10.8333 17.2788 10.8333 17.4998Z" fill="#D9D9D9" stroke="#D9D9D9" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M16.6667 17.4998C16.6667 17.7209 16.7545 17.9328 16.9108 18.0891C17.0671 18.2454 17.2791 18.3332 17.5001 18.3332C17.7211 18.3332 17.9331 18.2454 18.0893 18.0891C18.2456 17.9328 18.3334 17.7209 18.3334 17.4998C18.3334 17.2788 18.2456 17.0669 18.0893 16.9106C17.9331 16.7543 17.7211 16.6665 17.5001 16.6665C17.2791 16.6665 17.0671 16.7543 16.9108 16.9106C16.7545 17.0669 16.6667 17.2788 16.6667 17.4998Z" fill="#D9D9D9" stroke="#D9D9D9" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M22.5 17.4998C22.5 17.7209 22.5878 17.9328 22.7441 18.0891C22.9004 18.2454 23.1123 18.3332 23.3333 18.3332C23.5543 18.3332 23.7663 18.2454 23.9226 18.0891C24.0789 17.9328 24.1667 17.7209 24.1667 17.4998C24.1667 17.2788 24.0789 17.0669 23.9226 16.9106C23.7663 16.7543 23.5543 16.6665 23.3333 16.6665C23.1123 16.6665 22.9004 16.7543 22.7441 16.9106C22.5878 17.0669 22.5 17.2788 22.5 17.4998Z" fill="#D9D9D9" stroke="#D9D9D9" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_9916_1805">
+<rect width="20" height="20" fill="white" transform="translate(7.5 7.5)"/>
+</clipPath>
+</defs>
+</svg>

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/Contents.json

@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "filename" : "digest-play-button.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-play-button.imageset/digest-play-button.svg

@@ -0,0 +1,11 @@
+<svg width="35" height="35" viewBox="0 0 35 35" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="34" height="34" rx="17" stroke="#3D3D3D"/>
+<g clip-path="url(#clip0_9916_1819)">
+<path d="M13.75 12.5002V22.5002C13.75 22.6114 13.7796 22.7206 13.8359 22.8165C13.8921 22.9124 13.9729 22.9916 14.07 23.0459C14.1671 23.1002 14.2769 23.1275 14.3881 23.1252C14.4992 23.1229 14.6078 23.0909 14.7025 23.0327L22.8275 18.0327C22.9185 17.9768 22.9937 17.8985 23.0458 17.8052C23.0979 17.712 23.1253 17.607 23.1253 17.5002C23.1253 17.3934 23.0979 17.2883 23.0458 17.1951C22.9937 17.1019 22.9185 17.0236 22.8275 16.9677L14.7025 11.9677C14.6078 11.9094 14.4992 11.8775 14.3881 11.8751C14.2769 11.8728 14.1671 11.9002 14.07 11.9545C13.9729 12.0087 13.8921 12.0879 13.8359 12.1838C13.7796 12.2798 13.75 12.389 13.75 12.5002Z" fill="#EDEDED"/>
+</g>
+<defs>
+<clipPath id="clip0_9916_1819">
+<rect width="15" height="15" fill="white" transform="translate(10 10)"/>
+</clipPath>
+</defs>
+</svg>

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/Contents.json

@@ -0,0 +1,24 @@
+{
+  "images" : [
+    {
+      "filename" : "digest-trash-button.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  },
+  "properties" : {
+    "template-rendering-intent" : "template"
+  }
+}

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/digest-trash-button.imageset/digest-trash-button.svg

@@ -0,0 +1,15 @@
+<svg width="35" height="35" viewBox="0 0 35 35" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="0.5" y="0.5" width="34" height="34" rx="17" stroke="#3D3D3D"/>
+<g clip-path="url(#clip0_9916_1792)">
+<path d="M10.8333 13.3335H24.1666" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M15.8333 16.6665V21.6665" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M19.1667 16.6665V21.6665" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M11.6667 13.3335L12.5001 23.3335C12.5001 23.7755 12.6757 24.1994 12.9882 24.512C13.3008 24.8246 13.7247 25.0002 14.1667 25.0002H20.8334C21.2754 25.0002 21.6994 24.8246 22.0119 24.512C22.3245 24.1994 22.5001 23.7755 22.5001 23.3335L23.3334 13.3335" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M15 13.3333V10.8333C15 10.6123 15.0878 10.4004 15.2441 10.2441C15.4004 10.0878 15.6123 10 15.8333 10H19.1667C19.3877 10 19.5996 10.0878 19.7559 10.2441C19.9122 10.4004 20 10.6123 20 10.8333V13.3333" stroke="#D9D9D9" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
+</g>
+<defs>
+<clipPath id="clip0_9916_1792">
+<rect width="20" height="20" fill="white" transform="translate(7.5 7.5)"/>
+</clipPath>
+</defs>
+</svg>

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/Contents.json

@@ -0,0 +1,21 @@
+{
+  "images" : [
+    {
+      "filename" : "double_chevron_up.svg",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/double_chevron_up.imageset/double_chevron_up.svg

@@ -0,0 +1,4 @@
+<svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M17.4614 17L12.4614 12L7.46143 17" stroke="#898989" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M17.4614 11L12.4614 6L7.46143 11" stroke="#898989" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>

apple/OmnivoreKit/Sources/Views/Resources/zh-Hans.lproj/Localizable.strings

@@ -198,6 +198,7 @@
 "clubsGeneric" = "阅读俱乐部";
 "filterGeneric" = "特色功能";
 "errorGeneric" = "哦！出现了小问题，请您重试。";
+"readerSettingsGeneric" = "阅读设置";
 "pushNotificationsGeneric" = "推送通知";
 "dismissButton" = "返回或撤销";
 "errorNetwork" = "我们在连接到互联网时遇到问题。";

apple/OmnivoreKit/Sources/Views/Theme.swift

@@ -31,6 +31,22 @@ public enum Theme: String, CaseIterable {
     }
   }
 
+  public var fgColor: Color {
+    let prefersHighContrastText = UserDefaults.standard.bool(forKey: UserDefaultKey.prefersHighContrastWebFont.rawValue)
+    switch self {
+    case .system:
+      return Color.isDarkMode ? .white : .black
+    case .light:
+      return .black
+    case .dark:
+      return Color.white
+    case .sepia:
+      return prefersHighContrastText ? Color.black : (Color(hex: "#5F4B32") ?? Color.black)
+    case .apollo:
+      return prefersHighContrastText ? Color.white : (Color(hex: "#F3F3F3") ?? Color.white)
+    }
+  }
+
   public var toolbarColor: Color {
     ThemeManager.currentTheme.isDark ? Color.themeDarkWhiteGray : Color.themeMiddleGray
   }
@@ -98,6 +114,10 @@ public enum ThemeManager {
     currentTheme.bgColor
   }
 
+  public static var currentFgColor: Color {
+    currentTheme.fgColor
+  }
+
   public static var currentHighlightColor: Color {
     currentTheme.highlightColor
   }

packages/api/package.json

@@ -113,7 +113,8 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "word-counting": "^1.1.4",
-    "youtubei": "1.3.7"
+    "yaml": "^2.4.1",
+    "youtubei": "1.4.0"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",
@@ -124,7 +125,7 @@
     "@types/chai-as-promised": "^7.1.5",
     "@types/chai-string": "^1.4.2",
     "@types/cookie": "^0.4.0",
-    "@types/cookie-parser": "^1.4.2",
+    "@types/cookie-parser": "^1.4.7",
     "@types/csv-stringify": "^3.1.0",
     "@types/diff-match-patch": "^1.0.32",
     "@types/dompurify": "^2.0.4",

packages/api/src/apollo.ts

@@ -28,6 +28,11 @@ import { SetClaimsRole } from './utils/dictionary'
 import { logger } from './utils/logger'
 import { ReadingProgressDataSource } from './datasources/reading_progress_data_source'
 import { createPrometheusExporterPlugin } from '@bmatei/apollo-prometheus-exporter'
+import { ApolloServerPlugin } from 'apollo-server-plugin-base'
+import {
+  countDailyServiceUsage,
+  createServiceUsage,
+} from './services/service_usage'
 
 const signToken = promisify(jwt.sign)
 const pubsub = createPubSubClient()
@@ -112,10 +117,59 @@ export function makeApolloServer(app: Express): ApolloServer {
     },
   })
 
+  // enforce usage limits for the API
+  const usageLimitPlugin = (): ApolloServerPlugin<RequestContext> => {
+    // TODO: load the limit from the DB into memory when the server starts
+    // hardcode the limit for now
+    const MAX_SENT_EMAIL_PER_DAY = 3
+
+    return {
+      async requestDidStart(contextValue) {
+        // get graphql query from the request
+        const query = contextValue.request.query
+        // get the user id from the claims
+        const userId = contextValue.context.claims?.uid
+        const action = 'replyToEmail'
+        if (userId && query?.includes(action)) {
+          logger.info('checking usage limit for user', { userId, action })
+          // get the user's email sent count from the DB
+          const emailSentCount = await countDailyServiceUsage(userId, action)
+          if (emailSentCount >= MAX_SENT_EMAIL_PER_DAY) {
+            logger.info('user has reached the daily email limit', {
+              userId,
+              action,
+            })
+            // if the user has reached the limit, throw an error
+            throw new Error('You have reached the daily email limit')
+          }
+        }
+
+        return {
+          // track usage of the API
+          async willSendResponse(requestContext) {
+            // if the request was successful, increment the user's email sent count
+            if (
+              userId &&
+              query?.includes(action) &&
+              !requestContext.response.errors &&
+              !requestContext.response.data?.replyToEmail?.errorCodes
+            ) {
+              logger.info('incrementing usage count for user', {
+                userId,
+                action,
+              })
+              await createServiceUsage(userId, action)
+            }
+          },
+        }
+      },
+    }
+  }
+
   const apollo = new ApolloServer({
     schema: schema,
     context: contextFunc,
-    plugins: [promExporter],
+    plugins: [promExporter, usageLimitPlugin],
     formatError: (err) => {
       logger.info('server error', err)
       Sentry.captureException(err)
@@ -124,6 +178,7 @@ export function makeApolloServer(app: Express): ApolloServer {
     },
     introspection: env.dev.isLocal,
     persistedQueries: false,
+    stopOnTerminationSignals: false, // we handle this ourselves
   })
 
   return apollo

packages/api/src/entity/received_email.ts

@@ -33,6 +33,12 @@ export class ReceivedEmail {
   @Column('text')
   html!: string
 
+  @Column('text')
+  replyTo?: string
+
+  @Column('text')
+  reply?: string
+
   @Column('text')
   type!: 'article' | 'non-article'
 

packages/api/src/entity/service_usage.ts

@@ -0,0 +1,25 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+} from 'typeorm'
+import { User } from './user'
+
+@Entity('service_usage')
+export class ServiceUsage {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @ManyToOne(() => User)
+  @JoinColumn({ name: 'user_id' })
+  user!: User
+
+  @Column('varchar')
+  action!: string
+
+  @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' })
+  createdAt!: Date
+}

packages/api/src/generated/graphql.ts

@@ -58,6 +58,13 @@ export type AddPopularReadSuccess = {
   pageId: Scalars['String'];
 };
 
+export enum AllowedReply {
+  Confirm = 'CONFIRM',
+  Okay = 'OKAY',
+  Subscribe = 'SUBSCRIBE',
+  Yes = 'YES'
+}
+
 export type ApiKey = {
   __typename?: 'ApiKey';
   createdAt: Scalars['Date'];
@@ -1616,6 +1623,7 @@ export type Mutation = {
   optInFeature: OptInFeatureResult;
   recommend: RecommendResult;
   recommendHighlights: RecommendHighlightsResult;
+  replyToEmail: ReplyToEmailResult;
   reportItem: ReportItemResult;
   revokeApiKey: RevokeApiKeyResult;
   saveArticleReadingProgress: SaveArticleReadingProgressResult;
@@ -1836,6 +1844,12 @@ export type MutationRecommendHighlightsArgs = {
 };
 
 
+export type MutationReplyToEmailArgs = {
+  recentEmailId: Scalars['ID'];
+  reply: AllowedReply;
+};
+
+
 export type MutationReportItemArgs = {
   input: ReportItemInput;
 };
@@ -2275,6 +2289,8 @@ export type RecentEmail = {
   from: Scalars['String'];
   html?: Maybe<Scalars['String']>;
   id: Scalars['ID'];
+  reply?: Maybe<Scalars['String']>;
+  replyTo?: Maybe<Scalars['String']>;
   subject: Scalars['String'];
   text: Scalars['String'];
   to: Scalars['String'];
@@ -2430,6 +2446,22 @@ export type ReminderSuccess = {
   reminder: Reminder;
 };
 
+export type ReplyToEmailError = {
+  __typename?: 'ReplyToEmailError';
+  errorCodes: Array<ReplyToEmailErrorCode>;
+};
+
+export enum ReplyToEmailErrorCode {
+  Unauthorized = 'UNAUTHORIZED'
+}
+
+export type ReplyToEmailResult = ReplyToEmailError | ReplyToEmailSuccess;
+
+export type ReplyToEmailSuccess = {
+  __typename?: 'ReplyToEmailSuccess';
+  success: Scalars['Boolean'];
+};
+
 export type ReportItemInput = {
   itemUrl: Scalars['String'];
   pageId: Scalars['ID'];
@@ -3908,6 +3940,7 @@ export type ResolversTypes = {
   AddPopularReadErrorCode: AddPopularReadErrorCode;
   AddPopularReadResult: ResolversTypes['AddPopularReadError'] | ResolversTypes['AddPopularReadSuccess'];
   AddPopularReadSuccess: ResolverTypeWrapper<AddPopularReadSuccess>;
+  AllowedReply: AllowedReply;
   ApiKey: ResolverTypeWrapper<ApiKey>;
   ApiKeysError: ResolverTypeWrapper<ApiKeysError>;
   ApiKeysErrorCode: ApiKeysErrorCode;
@@ -4245,6 +4278,10 @@ export type ResolversTypes = {
   ReminderErrorCode: ReminderErrorCode;
   ReminderResult: ResolversTypes['ReminderError'] | ResolversTypes['ReminderSuccess'];
   ReminderSuccess: ResolverTypeWrapper<ReminderSuccess>;
+  ReplyToEmailError: ResolverTypeWrapper<ReplyToEmailError>;
+  ReplyToEmailErrorCode: ReplyToEmailErrorCode;
+  ReplyToEmailResult: ResolversTypes['ReplyToEmailError'] | ResolversTypes['ReplyToEmailSuccess'];
+  ReplyToEmailSuccess: ResolverTypeWrapper<ReplyToEmailSuccess>;
   ReportItemInput: ReportItemInput;
   ReportItemResult: ResolverTypeWrapper<ReportItemResult>;
   ReportType: ReportType;
@@ -4763,6 +4800,9 @@ export type ResolversParentTypes = {
   ReminderError: ReminderError;
   ReminderResult: ResolversParentTypes['ReminderError'] | ResolversParentTypes['ReminderSuccess'];
   ReminderSuccess: ReminderSuccess;
+  ReplyToEmailError: ReplyToEmailError;
+  ReplyToEmailResult: ResolversParentTypes['ReplyToEmailError'] | ResolversParentTypes['ReplyToEmailSuccess'];
+  ReplyToEmailSuccess: ReplyToEmailSuccess;
   ReportItemInput: ReportItemInput;
   ReportItemResult: ReportItemResult;
   RevokeApiKeyError: RevokeApiKeyError;
@@ -6128,6 +6168,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
   optInFeature?: Resolver<ResolversTypes['OptInFeatureResult'], ParentType, ContextType, RequireFields<MutationOptInFeatureArgs, 'input'>>;
   recommend?: Resolver<ResolversTypes['RecommendResult'], ParentType, ContextType, RequireFields<MutationRecommendArgs, 'input'>>;
   recommendHighlights?: Resolver<ResolversTypes['RecommendHighlightsResult'], ParentType, ContextType, RequireFields<MutationRecommendHighlightsArgs, 'input'>>;
+  replyToEmail?: Resolver<ResolversTypes['ReplyToEmailResult'], ParentType, ContextType, RequireFields<MutationReplyToEmailArgs, 'recentEmailId' | 'reply'>>;
   reportItem?: Resolver<ResolversTypes['ReportItemResult'], ParentType, ContextType, RequireFields<MutationReportItemArgs, 'input'>>;
   revokeApiKey?: Resolver<ResolversTypes['RevokeApiKeyResult'], ParentType, ContextType, RequireFields<MutationRevokeApiKeyArgs, 'id'>>;
   saveArticleReadingProgress?: Resolver<ResolversTypes['SaveArticleReadingProgressResult'], ParentType, ContextType, RequireFields<MutationSaveArticleReadingProgressArgs, 'input'>>;
@@ -6293,6 +6334,8 @@ export type RecentEmailResolvers<ContextType = ResolverContext, ParentType exten
   from?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   html?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  reply?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  replyTo?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   subject?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   text?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   to?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
@@ -6417,6 +6460,20 @@ export type ReminderSuccessResolvers<ContextType = ResolverContext, ParentType e
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type ReplyToEmailErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ReplyToEmailError'] = ResolversParentTypes['ReplyToEmailError']> = {
+  errorCodes?: Resolver<Array<ResolversTypes['ReplyToEmailErrorCode']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type ReplyToEmailResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ReplyToEmailResult'] = ResolversParentTypes['ReplyToEmailResult']> = {
+  __resolveType: TypeResolveFn<'ReplyToEmailError' | 'ReplyToEmailSuccess', ParentType, ContextType>;
+};
+
+export type ReplyToEmailSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ReplyToEmailSuccess'] = ResolversParentTypes['ReplyToEmailSuccess']> = {
+  success?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type ReportItemResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ReportItemResult'] = ResolversParentTypes['ReportItemResult']> = {
   message?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -7488,6 +7545,9 @@ export type Resolvers<ContextType = ResolverContext> = {
   ReminderError?: ReminderErrorResolvers<ContextType>;
   ReminderResult?: ReminderResultResolvers<ContextType>;
   ReminderSuccess?: ReminderSuccessResolvers<ContextType>;
+  ReplyToEmailError?: ReplyToEmailErrorResolvers<ContextType>;
+  ReplyToEmailResult?: ReplyToEmailResultResolvers<ContextType>;
+  ReplyToEmailSuccess?: ReplyToEmailSuccessResolvers<ContextType>;
   ReportItemResult?: ReportItemResultResolvers<ContextType>;
   RevokeApiKeyError?: RevokeApiKeyErrorResolvers<ContextType>;
   RevokeApiKeyResult?: RevokeApiKeyResultResolvers<ContextType>;

packages/api/src/generated/schema.graphql

@@ -37,6 +37,13 @@ type AddPopularReadSuccess {
   pageId: String!
 }
 
+enum AllowedReply {
+  CONFIRM
+  OKAY
+  SUBSCRIBE
+  YES
+}
+
 type ApiKey {
   createdAt: Date!
   expiresAt: Date!
@@ -1454,6 +1461,7 @@ type Mutation {
   optInFeature(input: OptInFeatureInput!): OptInFeatureResult!
   recommend(input: RecommendInput!): RecommendResult!
   recommendHighlights(input: RecommendHighlightsInput!): RecommendHighlightsResult!
+  replyToEmail(recentEmailId: ID!, reply: AllowedReply!): ReplyToEmailResult!
   reportItem(input: ReportItemInput!): ReportItemResult!
   revokeApiKey(id: ID!): RevokeApiKeyResult!
   saveArticleReadingProgress(input: SaveArticleReadingProgressInput!): SaveArticleReadingProgressResult!
@@ -1671,6 +1679,8 @@ type RecentEmail {
   from: String!
   html: String
   id: ID!
+  reply: String
+  replyTo: String
   subject: String!
   text: String!
   to: String!
@@ -1811,6 +1821,20 @@ type ReminderSuccess {
   reminder: Reminder!
 }
 
+type ReplyToEmailError {
+  errorCodes: [ReplyToEmailErrorCode!]!
+}
+
+enum ReplyToEmailErrorCode {
+  UNAUTHORIZED
+}
+
+union ReplyToEmailResult = ReplyToEmailError | ReplyToEmailSuccess
+
+type ReplyToEmailSuccess {
+  success: Boolean!
+}
+
 input ReportItemInput {
   itemUrl: String!
   pageId: ID!

packages/api/src/jobs/ai/create_digest.ts

@@ -0,0 +1,439 @@
+import { logger } from '../../utils/logger'
+import { v4 as uuid } from 'uuid'
+
+import { OpenAI } from '@langchain/openai'
+import { PromptTemplate } from '@langchain/core/prompts'
+import { LibraryItem } from '../../entity/library_item'
+import {
+  htmlToSpeechFile,
+  SpeechFile,
+  SSMLOptions,
+} from '@omnivore/text-to-speech-handler'
+import axios from 'axios'
+import {
+  findLibraryItemsByIds,
+  searchLibraryItems,
+} from '../../services/library_item'
+import { redisDataSource } from '../../redis_data_source'
+import { htmlToMarkdown } from '../../utils/parser'
+import yaml from 'yaml'
+import { JsonOutputParser } from '@langchain/core/output_parsers'
+import showdown from 'showdown'
+import { Digest, writeDigest } from '../../services/digest'
+import { TaskState } from '../../generated/graphql'
+
+export type CreateDigestJobSchedule = 'daily' | 'weekly'
+
+export interface CreateDigestJobData {
+  id: string
+  userId: string
+  voices?: string[]
+  language?: string
+  rate?: string
+  libraryItemIds?: string[]
+}
+
+export interface CreateDigestJobResponse {
+  jobId: string
+  jobState: TaskState
+}
+interface Selector {
+  query: string
+  count: number
+  reason: string
+}
+
+interface ZeroShotDefinition {
+  userPreferencesProfilePrompt: string
+  rankPrompt: string
+}
+
+interface DigestDefinition {
+  name: string
+  preferenceSelectors: Selector[]
+  candidateSelectors: Selector[]
+  contentFeaturesPrompt: string
+  contentRatingPrompt: string
+  summaryPrompt: string
+  assemblePrompt: string
+
+  zeroShot: ZeroShotDefinition
+}
+
+interface RankedItem {
+  topic: string
+  summary: string
+  libraryItem: LibraryItem
+}
+
+interface RankedTitle {
+  topic: string
+  id: string
+  title: string
+}
+
+export const CREATE_DIGEST_JOB = 'create-digest'
+
+let digestDefinition: DigestDefinition
+
+const fetchDigestDefinition = async (): Promise<DigestDefinition> => {
+  const promptFileUrl = process.env.PROMPT_FILE_URL
+  if (!promptFileUrl) {
+    const msg = 'PROMPT_FILE_URL not set'
+    logger.error(msg)
+    throw new Error(msg)
+  }
+
+  // fetch the yaml file
+  const response = await axios.get<string>(promptFileUrl)
+
+  // parse the yaml file
+  return yaml.parse(response.data) as DigestDefinition
+}
+
+// Makes multiple DB queries and combines the results
+const getPreferencesList = async (userId: string): Promise<LibraryItem[]> => {
+  // use the queries from the digest definitions to lookup preferences
+  // There should be a list of multiple queries we use. For now we can
+  // hardcode these queries:
+  // - query: "in:all is:read OR has:highlights sort:updated-desc wordsCount:>=20"
+  //   count: 21
+  //   reason: "recently read or highlighted items that are not part of the digest"
+  // - query: "in:all is:read OR has:highlights sort:saved-asc wordsCount:>=20"
+  //   count: 4
+  //   reason: "some older items that were interacted with"
+
+  const preferences = await Promise.all(
+    digestDefinition.preferenceSelectors.map(async (selector) => {
+      // use the selector to fetch items
+      const results = await searchLibraryItems(
+        {
+          query: selector.query,
+          size: selector.count,
+        },
+        userId
+      )
+
+      return results.libraryItems
+    })
+  )
+
+  // deduplicate and flatten the items
+  const dedupedPreferences = preferences
+    .flat()
+    .filter(
+      (item, index, self) => index === self.findIndex((t) => t.id === item.id)
+    )
+
+  return dedupedPreferences
+}
+
+// Makes multiple DB queries and combines the results
+const getCandidatesList = async (
+  userId: string,
+  libraryItemIds?: string[]
+): Promise<LibraryItem[]> => {
+  // use the queries from the digest definitions to lookup preferences
+  // There should be a list of multiple queries we use. For now we can
+  // hardcode these queries:
+  // - query: "in:all is:unread saved:last24hrs sort:saved-desc wordsCount:>=500"
+  //   count: 100
+  //   reason: "most recent 100 items saved over 500 words
+
+  if (libraryItemIds) {
+    logger.info('Using libraryItemIds')
+    return findLibraryItemsByIds(libraryItemIds, userId)
+  }
+
+  const candidates = await Promise.all(
+    digestDefinition.candidateSelectors.map(async (selector) => {
+      // use the selector to fetch items
+      const results = await searchLibraryItems(
+        {
+          includeContent: true,
+          query: selector.query,
+          size: selector.count,
+        },
+        userId
+      )
+
+      return results.libraryItems
+    })
+  )
+
+  // deduplicate and flatten the items
+  const dedupedCandidates = candidates
+    .flat()
+    .filter(
+      (item, index, self) => index === self.findIndex((t) => t.id === item.id)
+    )
+    .map((item) => ({
+      ...item,
+      readableContent: htmlToMarkdown(item.readableContent),
+    })) // convert the html content to markdown
+
+  return dedupedCandidates
+}
+
+// Takes a list of library items, and uses a prompt to generate
+// a text representation of a user profile
+const createUserProfile = async (
+  preferences: LibraryItem[]
+): Promise<string> => {
+  const llm = new OpenAI({
+    modelName: 'gpt-4-0125-preview',
+    configuration: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  })
+
+  const contextualTemplate = PromptTemplate.fromTemplate(
+    digestDefinition.zeroShot.userPreferencesProfilePrompt
+  )
+
+  const chain = contextualTemplate.pipe(llm)
+  const result = await chain.invoke({
+    titles: preferences.map((item) => `* ${item.title}`).join('\n'),
+  })
+
+  return result
+}
+
+// Checks redis for a user profile, if not found creates one and writes
+// it to redis
+const findOrCreateUserProfile = async (userId: string): Promise<string> => {
+  // check redis for user profile, return if found
+  const key = `userProfile:${userId}`
+  const existingProfile = await redisDataSource.redisClient?.get(key)
+  if (existingProfile) {
+    return existingProfile
+  }
+
+  // if not found
+  const preferences = await getPreferencesList(userId)
+  const profile = await createUserProfile(preferences)
+
+  // write to redis here and ttl is 1 week
+  await redisDataSource.redisClient?.set(key, profile, 'EX', 60 * 60 * 24 * 7)
+
+  return profile
+}
+
+// Uses OpenAI to rank all the titles based on the user profiles
+const rankCandidates = async (
+  candidates: LibraryItem[],
+  userProfile: string
+): Promise<RankedItem[]> => {
+  const llm = new OpenAI({
+    modelName: 'gpt-4-0125-preview',
+    configuration: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  })
+
+  const contextualTemplate = PromptTemplate.fromTemplate(
+    digestDefinition.zeroShot.rankPrompt
+  )
+
+  const outputParser = new JsonOutputParser()
+  const chain = contextualTemplate.pipe(llm).pipe(outputParser)
+  const contextStr = await chain.invoke({
+    userProfile,
+    titles: JSON.stringify(
+      candidates.map((item) => ({
+        id: item.id,
+        title: item.title,
+      }))
+    ),
+  })
+
+  logger.info('contextStr: ', contextStr)
+  // convert the json output to an array of ranked candidates
+  const rankedCandidate = contextStr as RankedTitle[]
+
+  // map the ranked titles to the library items based on id
+  const rankedItems = rankedCandidate
+    .map((item) => {
+      const libraryItem = candidates.find((t) => t.id === item.id)
+      return {
+        topic: item.topic,
+        libraryItem,
+        summary: '',
+      }
+    })
+    .filter((item) => item.libraryItem !== undefined) as RankedItem[]
+
+  return rankedItems
+}
+
+// Does some grouping by topic while trying to maintain ranking
+// adds some basic topic diversity
+const chooseRankedSelections = (rankedCandidates: RankedItem[]) => {
+  const selected = []
+  const rankedTopics = []
+  const topicCount = {} as Record<string, number>
+
+  for (const item of rankedCandidates) {
+    if (selected.length >= 5) {
+      break
+    }
+
+    topicCount[item.topic] = (topicCount[item.topic] || 0) + 1
+
+    if (topicCount[item.topic] <= 2) {
+      selected.push(item)
+      if (rankedTopics.indexOf(item.topic) === -1) {
+        rankedTopics.push(item.topic)
+      }
+    }
+  }
+
+  logger.info('rankedTopics: ', rankedTopics)
+  logger.info('finalSelections: ', selected)
+
+  const finalSelections = []
+
+  for (const topic of rankedTopics) {
+    const matches = selected.filter((item) => item.topic == topic)
+    finalSelections.push(...matches)
+  }
+
+  logger.info('finalSelections: ', finalSelections)
+
+  return { finalSelections, rankedTopics }
+}
+
+const summarizeItems = async (
+  rankedCandidates: RankedItem[]
+): Promise<RankedItem[]> => {
+  const llm = new OpenAI({
+    modelName: 'gpt-4-0125-preview',
+    configuration: {
+      apiKey: process.env.OPENAI_API_KEY,
+    },
+  })
+
+  const contextualTemplate = PromptTemplate.fromTemplate(
+    digestDefinition.summaryPrompt
+  )
+  const chain = contextualTemplate.pipe(llm)
+
+  // send all the ranked candidates to openAI at once in a batch
+  const summaries = await chain.batch(
+    rankedCandidates.map((item) => ({
+      title: item.libraryItem.title,
+      author: item.libraryItem.author ?? '',
+      content: item.libraryItem.readableContent, // markdown content
+    }))
+  )
+
+  summaries.forEach(
+    (summary, index) => (rankedCandidates[index].summary = summary)
+  )
+
+  return rankedCandidates
+}
+
+// generate speech files from the summaries
+const generateSpeechFiles = (
+  rankedItems: RankedItem[],
+  options: SSMLOptions
+): SpeechFile[] => {
+  // convert the summaries from markdown to HTML
+  const converter = new showdown.Converter({
+    backslashEscapesHTMLTags: true,
+  })
+
+  const speechFiles = rankedItems.map((item) => {
+    const html = `
+    <div id="readability-content">
+      <div id="readability-page-1">
+        ${converter.makeHtml(item.summary)}
+      </div>
+    </div>`
+    return htmlToSpeechFile({
+      content: html,
+      options,
+    })
+  })
+
+  return speechFiles
+}
+
+// we should have a QA step here that does some
+// basic checks to make sure the summaries are good.
+const filterSummaries = (summaries: RankedItem[]): RankedItem[] => {
+  return summaries.filter((item) => item.summary.length > 100)
+}
+
+// we can use something more sophisticated to generate titles
+const generateTitle = (summaries: RankedItem[]): string =>
+  'Omnivore digest: ' +
+  summaries.map((item) => item.libraryItem.title).join(', ')
+
+// generate description based on the summaries
+const generateDescription = (
+  summaries: RankedItem[],
+  rankedTopics: string[]
+): string =>
+  `We selected ${
+    summaries.length
+  } articles from your last 24 hours of saved items, covering ${rankedTopics.join(
+    ', '
+  )}.`
+
+// generate content based on the summaries
+const generateContent = (summaries: RankedItem[]): string =>
+  summaries
+    .map((summary) => `### ${summary.libraryItem.title}\n ${summary.summary}`)
+    .join('\n\n')
+
+const generateByline = (summaries: RankedItem[]): string =>
+  summaries
+    .filter((summary) => !!summary.libraryItem.author)
+    .map((item) => item.libraryItem.author)
+    .join(', ')
+
+export const createDigestJob = async (jobData: CreateDigestJobData) => {
+  digestDefinition = await fetchDigestDefinition()
+
+  const candidates = await getCandidatesList(
+    jobData.userId,
+    jobData.libraryItemIds
+  )
+  const userProfile = await findOrCreateUserProfile(jobData.userId)
+  const rankedCandidates = await rankCandidates(candidates, userProfile)
+  const { finalSelections, rankedTopics } =
+    chooseRankedSelections(rankedCandidates)
+
+  const summaries = await summarizeItems(finalSelections)
+
+  const filteredSummaries = filterSummaries(summaries)
+
+  const speechFiles = generateSpeechFiles(filteredSummaries, {
+    ...jobData,
+    primaryVoice: jobData.voices?.[0],
+    secondaryVoice: jobData.voices?.[1],
+  })
+  const title = generateTitle(summaries)
+  const digest: Digest = {
+    id: jobData.id,
+    title,
+    content: generateContent(summaries),
+    urlsToAudio: [],
+    jobState: TaskState.Succeeded,
+    speechFiles,
+    chapters: filteredSummaries.map((item, index) => ({
+      title: item.libraryItem.title,
+      id: item.libraryItem.id,
+      url: item.libraryItem.originalUrl,
+      thumbnail: item.libraryItem.thumbnail ?? undefined,
+      wordCount: speechFiles[index].wordCount,
+    })),
+    createdAt: new Date(),
+    description: generateDescription(summaries, rankedTopics),
+    byline: generateByline(summaries),
+  }
+
+  await writeDigest(jobData.userId, digest)
+}

packages/api/src/jobs/email/inbound_emails.ts

@@ -0,0 +1,304 @@
+import { handleNewsletter } from '@omnivore/content-handler'
+import { Converter } from 'showdown'
+import { ContentReaderType, LibraryItemState } from '../../entity/library_item'
+import { SubscriptionStatus } from '../../entity/subscription'
+import { UploadFile } from '../../entity/upload_file'
+import { env } from '../../env'
+import { PageType, UploadFileStatus } from '../../generated/graphql'
+import { authTrx } from '../../repository'
+import { createOrUpdateLibraryItem } from '../../services/library_item'
+import {
+  findNewsletterEmailByAddress,
+  updateConfirmationCode,
+} from '../../services/newsletters'
+import {
+  saveReceivedEmail,
+  updateReceivedEmail,
+} from '../../services/received_emails'
+import { saveNewsletter } from '../../services/save_newsletter_email'
+import { saveUrlFromEmail } from '../../services/save_url'
+import { getSubscriptionByName } from '../../services/subscriptions'
+import { analytics } from '../../utils/analytics'
+import { enqueueSendEmail } from '../../utils/createTask'
+import { generateSlug, isUrl } from '../../utils/helpers'
+import { logger } from '../../utils/logger'
+import {
+  parseEmailAddress,
+  isProbablyArticle,
+  getTitleFromEmailSubject,
+  generateUniqueUrl,
+} from '../../utils/parser'
+import {
+  generateUploadFilePathName,
+  getStorageFileDetails,
+} from '../../utils/uploads'
+
+interface EmailJobData {
+  from: string
+  to: string
+  subject: string
+  html: string
+  text: string
+  headers: Record<string, string | string[]>
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+  forwardedFrom?: string
+  replyTo?: string
+  confirmationCode?: string
+  uploadFile?: {
+    fileName: string
+    contentType: string
+    id: string
+  }
+}
+
+const converter = new Converter()
+export const FORWARD_EMAIL_JOB = 'forward-email'
+export const SAVE_NEWSLETTER_JOB = 'save-newsletter'
+export const CONFIRM_EMAIL_JOB = 'confirmation-email'
+export const SAVE_ATTACHMENT_JOB = 'save-attachment'
+
+export const plainTextToHtml = (text: string): string => {
+  return converter.makeHtml(text)
+}
+
+export const forwardEmailJob = async (data: EmailJobData) => {
+  const { from, to, subject, html, text, replyTo, forwardedFrom } = data
+
+  // get user from newsletter email
+  const newsletterEmail = await findNewsletterEmailByAddress(to)
+
+  if (!newsletterEmail) {
+    logger.error(`newsletter email not found: ${to}`)
+    return false
+  }
+
+  const user = newsletterEmail.user
+  const parsedFrom = parseEmailAddress(from)
+
+  const { id: receivedEmailId } = await saveReceivedEmail(
+    from,
+    to,
+    subject,
+    text,
+    html,
+    user.id,
+    'non-article',
+    replyTo
+  )
+
+  if (
+    await isProbablyArticle(
+      forwardedFrom || parsedFrom.address || from,
+      subject
+    )
+  ) {
+    logger.info('handling as article')
+    return saveNewsletter(
+      {
+        title: getTitleFromEmailSubject(subject),
+        author: parsedFrom.name || from,
+        url: generateUniqueUrl(),
+        content: html || text,
+        receivedEmailId,
+        email: newsletterEmail.address,
+      },
+      newsletterEmail
+    )
+  }
+
+  analytics.capture({
+    distinctId: user.id,
+    event: 'non_newsletter_email_received',
+    properties: {
+      env: env.server.apiEnv,
+    },
+  })
+
+  // forward non-newsletter emails to the registered email address
+  const result = await enqueueSendEmail({
+    from: env.sender.message,
+    to: user.email,
+    subject: `Fwd: ${subject}`,
+    html,
+    text,
+    replyTo: replyTo || from,
+  })
+
+  return !!result
+}
+
+export const saveNewsletterJob = async (data: EmailJobData) => {
+  const {
+    from,
+    to,
+    subject,
+    html,
+    text,
+    replyTo,
+    headers,
+    unsubMailTo,
+    unsubHttpUrl,
+  } = data
+
+  // get user from newsletter email
+  const newsletterEmail = await findNewsletterEmailByAddress(to)
+  if (!newsletterEmail) {
+    logger.error(`newsletter email not found: ${to}`)
+
+    return false
+  }
+
+  const user = newsletterEmail.user
+  const { id: receivedEmailId } = await saveReceivedEmail(
+    from,
+    to,
+    subject,
+    text,
+    html,
+    user.id,
+    'non-article', // default to non-article
+    replyTo
+  )
+
+  if (isUrl(subject)) {
+    // save url if the title is a parsable url
+    const result = await saveUrlFromEmail(
+      subject,
+      receivedEmailId,
+      newsletterEmail.user.id
+    )
+
+    if (result) {
+      // update received email type
+      await updateReceivedEmail(receivedEmailId, 'article', user.id)
+    }
+
+    return result
+  }
+
+  // convert text to html if html is not available
+  const content = html || plainTextToHtml(text)
+  const newsletter = await handleNewsletter({
+    from,
+    to,
+    subject,
+    html: content,
+    headers,
+  })
+
+  const parsedFrom = parseEmailAddress(from)
+  const author = parsedFrom.name || from
+
+  // do not subscribe if subscription already exists and is unsubscribed
+  const existingSubscription = await getSubscriptionByName(
+    author,
+    newsletterEmail.user.id
+  )
+  if (existingSubscription?.status === SubscriptionStatus.Unsubscribed) {
+    logger.info(`newsletter already unsubscribed: ${from}`)
+    return false
+  }
+
+  // save newsletter instead
+  return saveNewsletter(
+    {
+      email: newsletterEmail.address,
+      content,
+      url: generateUniqueUrl(),
+      title: subject,
+      author,
+      unsubMailTo,
+      unsubHttpUrl,
+      receivedEmailId,
+      ...newsletter,
+    },
+    newsletterEmail
+  )
+}
+
+export const saveAttachmentJob = async (data: EmailJobData) => {
+  const { from, to, subject, html, text, replyTo, uploadFile } = data
+
+  // get user from newsletter email
+  const newsletterEmail = await findNewsletterEmailByAddress(to)
+  if (!newsletterEmail) {
+    logger.error(`newsletter email not found: ${to}`)
+
+    return false
+  }
+
+  const user = newsletterEmail.user
+  const receivedEmail = await saveReceivedEmail(
+    from,
+    to,
+    subject,
+    text,
+    html,
+    user.id,
+    'non-article',
+    replyTo
+  )
+
+  const uploadFileData = await authTrx(
+    (tx) =>
+      tx.getRepository(UploadFile).save({
+        ...uploadFile,
+        url: '', // no url for email attachments
+        status: UploadFileStatus.Completed,
+        user: { id: user.id },
+      }),
+    undefined,
+    user.id
+  )
+
+  const uploadFileDetails = await getStorageFileDetails(
+    uploadFileData.id,
+    uploadFileData.fileName
+  )
+
+  const uploadFilePathName = generateUploadFilePathName(
+    uploadFileData.id,
+    uploadFileData.fileName
+  )
+
+  const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFilePathName}`
+  const uploadFileHash = uploadFileDetails.md5Hash
+  const itemType =
+    uploadFileData.contentType === 'application/pdf'
+      ? PageType.File
+      : PageType.Book
+  const title = subject || uploadFileData.fileName
+  const itemToCreate = {
+    originalUrl: uploadFileUrlOverride,
+    itemType,
+    textContentHash: uploadFileHash,
+    uploadFile: { id: uploadFileData.id },
+    title,
+    readableContent: '',
+    slug: generateSlug(title),
+    state: LibraryItemState.Succeeded,
+    user: { id: user.id },
+    contentReader:
+      itemType === PageType.File
+        ? ContentReaderType.PDF
+        : ContentReaderType.EPUB,
+  }
+
+  await createOrUpdateLibraryItem(itemToCreate, user.id)
+
+  // update received email type
+  await updateReceivedEmail(receivedEmail.id, 'article', user.id)
+
+  return true
+}
+
+export const confirmEmailJob = async (data: EmailJobData) => {
+  const { confirmationCode, to } = data
+  if (!confirmationCode) {
+    logger.error('confirmation code not provided')
+    return false
+  }
+
+  return updateConfirmationCode(to, confirmationCode)
+}

packages/api/src/jobs/email/send_email.ts

@@ -1,27 +1,29 @@
-import { env } from '../env'
-import { sendWithMailJet } from '../services/send_emails'
-import { Merge } from '../util'
-import { logger } from '../utils/logger'
-import { sendEmail } from '../utils/sendEmail'
+import { env } from '../../env'
+import { sendWithMailJet } from '../../services/send_emails'
+import { Merge } from '../../util'
+import { logger } from '../../utils/logger'
+import { sendEmail } from '../../utils/sendEmail'
 
 export const SEND_EMAIL_JOB = 'send-email'
 
 type ContentType = { html: string } | { text: string } | { templateId: string }
 export type SendEmailJobData = Merge<
   {
-    emailAddress: string
+    to: string
+    from?: string
     subject?: string
     html?: string
     text?: string
     templateId?: string
     dynamicTemplateData?: Record<string, any>
+    replyTo?: string
   },
   ContentType
 >
 
 export const sendEmailJob = async (data: SendEmailJobData) => {
   if (process.env.USE_MAILJET && data.dynamicTemplateData) {
-    return sendWithMailJet(data.emailAddress, data.dynamicTemplateData.link)
+    return sendWithMailJet(data.to, data.dynamicTemplateData.link)
   }
 
   if (!data.html && !data.text && !data.templateId) {
@@ -31,7 +33,6 @@ export const sendEmailJob = async (data: SendEmailJobData) => {
 
   return sendEmail({
     ...data,
-    from: env.sender.message,
-    to: data.emailAddress,
+    from: data.from || env.sender.message,
   })
 }

packages/api/src/jobs/rss/refreshFeed.ts

@@ -192,22 +192,40 @@ export const fetchAndChecksum = async (url: string) => {
 
 const parseFeed = async (url: string, content: string) => {
   try {
-    // check if url is a telegram channel
-    const telegramRegex = /https:\/\/t\.me\/([a-zA-Z0-9_]+)/
+    // check if url is a telegram channel or preview
+    const telegramRegex = /t\.me\/([^/]+)/
     const telegramMatch = url.match(telegramRegex)
     if (telegramMatch) {
+      let channel = telegramMatch[1]
+      if (channel.startsWith('s/')) {
+        channel = channel.slice(2)
+      } else {
+        // open the preview page to get the data
+        const fetchResult = await fetchAndChecksum(`https://t.me/s/${channel}`)
+        if (!fetchResult) {
+          return null
+        }
+
+        content = fetchResult.content
+      }
+
       const dom = parseHTML(content).document
-      const title = dom.querySelector('meta[property="og:title"]')
+      const title =
+        dom
+          .querySelector('meta[property="og:title"]')
+          ?.getAttribute('content') || dom.title
       // post has attribute data-post
       const posts = dom.querySelectorAll('[data-post]')
       const items = Array.from(posts)
         .map((post) => {
-          const id = post.getAttribute('data-post')
+          const id = post.getAttribute('data-post')?.split('/')[1]
           if (!id) {
             return null
           }
 
-          const url = `https://t.me/${telegramMatch[1]}/${id}`
+          const url = `https://t.me/s/${channel}/${id}`
+          const content = post.outerHTML
+
           // find the <time> element
           const time = post.querySelector('time')
           const dateTime = time?.getAttribute('datetime') || undefined
@@ -215,12 +233,16 @@ const parseFeed = async (url: string, content: string) => {
           return {
             link: url,
             isoDate: dateTime,
+            title: `${title} - ${id}`,
+            creator: title,
+            content,
+            links: [url],
           }
         })
         .filter((item) => !!item) as RssFeedItem[]
 
       return {
-        title: title?.getAttribute('content') || dom.title,
+        title,
         items,
       }
     }
@@ -511,7 +533,7 @@ const processSubscription = async (
 
   // fetch feed
   let itemCount = 0,
-    failedAt: Date | undefined
+    failedAt: Date | null = null
 
   const feedLastBuildDate = feed.lastBuildDate
   logger.info(`Feed last build date ${feedLastBuildDate || 'N/A'}`)

packages/api/src/queue-processor.ts

@@ -16,6 +16,7 @@ import { appDataSource } from './data_source'
 import { env } from './env'
 import { TaskState } from './generated/graphql'
 import { aiSummarize, AI_SUMMARIZE_JOB_NAME } from './jobs/ai-summarize'
+import { createDigestJob, CREATE_DIGEST_JOB } from './jobs/ai/create_digest'
 import { bulkAction, BULK_ACTION_JOB_NAME } from './jobs/bulk_action'
 import { callWebhook, CALL_WEBHOOK_JOB_NAME } from './jobs/call_webhook'
 import { findThumbnail, THUMBNAIL_JOB } from './jobs/find_thumbnail'
@@ -36,7 +37,7 @@ import {
 import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
 import { refreshFeed } from './jobs/rss/refreshFeed'
 import { savePageJob } from './jobs/save_page'
-import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/send_email'
+import { sendEmailJob, SEND_EMAIL_JOB } from './jobs/email/send_email'
 import {
   syncReadPositionsJob,
   SYNC_READ_POSITIONS_JOB_NAME,
@@ -53,20 +54,28 @@ import { redisDataSource } from './redis_data_source'
 import { CACHED_READING_POSITION_PREFIX } from './services/cached_reading_position'
 import { getJobPriority } from './utils/createTask'
 import { logger } from './utils/logger'
+import {
+  confirmEmailJob,
+  CONFIRM_EMAIL_JOB,
+  forwardEmailJob,
+  FORWARD_EMAIL_JOB,
+  saveAttachmentJob,
+  saveNewsletterJob,
+  SAVE_ATTACHMENT_JOB,
+  SAVE_NEWSLETTER_JOB,
+} from './jobs/email/inbound_emails'
 
 export const QUEUE_NAME = 'omnivore-backend-queue'
 export const JOB_VERSION = 'v001'
 
-let backendQueue: Queue | undefined
-export const getBackendQueue = async (): Promise<Queue | undefined> => {
-  if (backendQueue) {
-    await backendQueue.waitUntilReady()
-    return backendQueue
-  }
+export const getBackendQueue = async (
+  name = QUEUE_NAME
+): Promise<Queue | undefined> => {
   if (!redisDataSource.workerRedisClient) {
     throw new Error('Can not create queues, redis is not initialized')
   }
-  backendQueue = new Queue(QUEUE_NAME, {
+
+  const backendQueue = new Queue(name, {
     connection: redisDataSource.workerRedisClient,
     defaultJobOptions: {
       backoff: {
@@ -85,8 +94,11 @@ export const getBackendQueue = async (): Promise<Queue | undefined> => {
   return backendQueue
 }
 
-export const getJob = async (jobId: string) => {
-  const queue = await getBackendQueue()
+export const createJobId = (jobName: string, userId: string) =>
+  `${jobName}_${userId}_${JOB_VERSION}`
+
+export const getJob = async (jobId: string, queueName?: string) => {
+  const queue = await getBackendQueue(queueName)
   if (!queue) {
     return
   }
@@ -160,6 +172,16 @@ export const createWorker = (connection: ConnectionOptions) =>
           return exportAllItems(job.data)
         case SEND_EMAIL_JOB:
           return sendEmailJob(job.data)
+        case CONFIRM_EMAIL_JOB:
+          return confirmEmailJob(job.data)
+        case SAVE_ATTACHMENT_JOB:
+          return saveAttachmentJob(job.data)
+        case SAVE_NEWSLETTER_JOB:
+          return saveNewsletterJob(job.data)
+        case FORWARD_EMAIL_JOB:
+          return forwardEmailJob(job.data)
+        case CREATE_DIGEST_JOB:
+          return createDigestJob(job.data)
         default:
           logger.warning(`[queue-processor] unhandled job: ${job.name}`)
       }
@@ -304,8 +326,19 @@ const main = async () => {
 
   const gracefulShutdown = async (signal: string) => {
     console.log(`[queue-processor]: Received ${signal}, closing server...`)
+    await new Promise<void>((resolve) => {
+      server.close((err) => {
+        console.log('[queue-processor]: Express server closed')
+        if (err) {
+          console.log('[queue-processor]: error stopping server', { err })
+        }
+
+        resolve()
+      })
+    })
     await worker.close()
     await redisDataSource.shutdown()
+    await appDataSource.destroy()
     process.exit(0)
   }
 

packages/api/src/repository/label.ts

@@ -46,9 +46,12 @@ export const labelRepository = appDataSource.getRepository(Label).extend({
     return this.findOneBy({ id })
   },
 
-  findByName(name: string) {
+  findByName(name: string, userId: string) {
     return this.createQueryBuilder()
-      .where('LOWER(name) = LOWER(:name)', { name }) // case insensitive
+      .where('user_id = :userId AND LOWER(name) = LOWER(:name)', {
+        name,
+        userId,
+      }) // case insensitive
       .getOne()
   },
 

packages/api/src/resolvers/function_resolvers.ts

@@ -150,7 +150,11 @@ import {
   webhookResolver,
   webhooksResolver,
 } from './index'
-import { markEmailAsItemResolver, recentEmailsResolver } from './recent_emails'
+import {
+  markEmailAsItemResolver,
+  recentEmailsResolver,
+  replyToEmailResolver,
+} from './recent_emails'
 import { recentSearchesResolver } from './recent_searches'
 import { WithDataSourcesContext } from './types'
 import { updateEmailResolver } from './user'
@@ -316,6 +320,7 @@ export const functionResolvers = {
     emptyTrash: emptyTrashResolver,
     fetchContent: fetchContentResolver,
     exportToIntegration: exportToIntegrationResolver,
+    replyToEmail: replyToEmailResolver,
   },
   Query: {
     me: getMeUserResolver,
@@ -368,6 +373,17 @@ export const functionResolvers = {
       }
       return undefined
     },
+    async features(
+      _: User,
+      __: Record<string, unknown>,
+      ctx: WithDataSourcesContext
+    ) {
+      if (!ctx.claims?.uid) {
+        return undefined
+      }
+
+      return []
+    },
     async featureList(
       _: User,
       __: Record<string, unknown>,
@@ -379,17 +395,6 @@ export const functionResolvers = {
 
       return findUserFeatures(ctx.claims.uid)
     },
-    async features(
-      user: User,
-      __: Record<string, unknown>,
-      ctx: WithDataSourcesContext
-    ) {
-      if (!ctx.claims?.uid) {
-        return undefined
-      }
-
-      return (await findUserFeatures(ctx.claims.uid)).map((f) => f.name)
-    },
   },
   Article: {
     async url(article: Article, _: unknown, ctx: WithDataSourcesContext) {
@@ -680,4 +685,5 @@ export const functionResolvers = {
   ...resultResolveTypeResolver('FetchContent'),
   ...resultResolveTypeResolver('Integration'),
   ...resultResolveTypeResolver('ExportToIntegration'),
+  ...resultResolveTypeResolver('ReplyToEmail'),
 }

packages/api/src/resolvers/highlight/index.ts

@@ -81,7 +81,7 @@ export const mergeHighlightResolver = authorized<
   MergeHighlightSuccess,
   MergeHighlightError,
   MutationMergeHighlightArgs
->(async (_, { input }, { log, pubsub, uid }) => {
+>(async (_, { input }, { authTrx, log, pubsub, uid }) => {
   const { overlapHighlightIdList, ...newHighlightInput } = input
 
   /* Compute merged annotation form the order of highlights appearing on page */
@@ -90,9 +90,10 @@ export const mergeHighlightResolver = authorized<
   const mergedColors: string[] = []
 
   try {
-    const existingHighlights = await highlightRepository.findByLibraryItemId(
-      input.articleId,
-      uid
+    const existingHighlights = await authTrx((tx) =>
+      tx
+        .withRepository(highlightRepository)
+        .findByLibraryItemId(input.articleId, uid)
     )
 
     existingHighlights.forEach((highlight) => {

packages/api/src/resolvers/labels/index.ts

@@ -90,17 +90,22 @@ export const createLabelResolver = authorized<
   CreateLabelError,
   MutationCreateLabelArgs
 >(async (_, { input }, { authTrx, uid }) => {
-  const existingLabel = await labelRepository.findByName(input.name)
-  if (existingLabel) {
+  const label = await authTrx(async (tx) => {
+    const repo = tx.withRepository(labelRepository)
+    const existingLabel = await repo.findByName(input.name, uid)
+    if (existingLabel) {
+      return null
+    }
+
+    return repo.createLabel(input, uid)
+  })
+
+  if (!label) {
     return {
       errorCodes: [CreateLabelErrorCode.LabelAlreadyExists],
     }
   }
 
-  const label = await authTrx(async (tx) =>
-    tx.withRepository(labelRepository).createLabel(input, uid)
-  )
-
   analytics.capture({
     distinctId: uid,
     event: 'label_created',

packages/api/src/resolvers/recent_emails/index.ts

@@ -7,12 +7,17 @@ import {
   MarkEmailAsItemErrorCode,
   MarkEmailAsItemSuccess,
   MutationMarkEmailAsItemArgs,
+  MutationReplyToEmailArgs,
   RecentEmailsError,
-  RecentEmailsErrorCode,
   RecentEmailsSuccess,
+  ReplyToEmailError,
+  ReplyToEmailErrorCode,
+  ReplyToEmailSuccess,
 } from '../../generated/graphql'
+import { getRepository } from '../../repository'
 import { updateReceivedEmail } from '../../services/received_emails'
 import { saveNewsletter } from '../../services/save_newsletter_email'
+import { enqueueSendEmail } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 import { generateUniqueUrl, parseEmailAddress } from '../../utils/parser'
 import { sendEmail } from '../../utils/sendEmail'
@@ -20,27 +25,19 @@ import { sendEmail } from '../../utils/sendEmail'
 export const recentEmailsResolver = authorized<
   RecentEmailsSuccess,
   RecentEmailsError
->(async (_, __, { authTrx, log, uid }) => {
-  try {
-    const recentEmails = await authTrx((t) =>
-      t.getRepository(ReceivedEmail).find({
-        where: {
-          user: { id: uid },
-        },
-        order: { createdAt: 'DESC' },
-        take: 20,
-      })
-    )
+>(async (_, __, { authTrx, uid }) => {
+  const recentEmails = await authTrx((t) =>
+    t.getRepository(ReceivedEmail).find({
+      where: {
+        user: { id: uid },
+      },
+      order: { createdAt: 'DESC' },
+      take: 20,
+    })
+  )
 
-    return {
-      recentEmails,
-    }
-  } catch (error) {
-    log.error('Error getting recent emails', error)
-
-    return {
-      errorCodes: [RecentEmailsErrorCode.BadRequest],
-    }
+  return {
+    recentEmails,
   }
 })
 
@@ -49,87 +46,114 @@ export const markEmailAsItemResolver = authorized<
   MarkEmailAsItemError,
   MutationMarkEmailAsItemArgs
 >(async (_, { recentEmailId }, { authTrx, uid, log }) => {
-  try {
-    const recentEmail = await authTrx((t) =>
-      t.getRepository(ReceivedEmail).findOneBy({
-        id: recentEmailId,
+  const recentEmail = await authTrx((t) =>
+    t.getRepository(ReceivedEmail).findOneBy({
+      id: recentEmailId,
+      user: { id: uid },
+      type: 'non-article',
+    })
+  )
+  if (!recentEmail) {
+    log.info('no recent email', recentEmailId)
+
+    return {
+      errorCodes: [MarkEmailAsItemErrorCode.Unauthorized],
+    }
+  }
+
+  const newsletterEmail = await authTrx((t) =>
+    t.getRepository(NewsletterEmail).findOne({
+      where: {
         user: { id: uid },
-        type: 'non-article',
-      })
-    )
-    if (!recentEmail) {
-      log.info('no recent email', recentEmailId)
-
-      return {
-        errorCodes: [MarkEmailAsItemErrorCode.Unauthorized],
-      }
-    }
-
-    const newsletterEmail = await authTrx((t) =>
-      t.getRepository(NewsletterEmail).findOne({
-        where: {
-          user: { id: uid },
-          address: ILike(recentEmail.to),
-        },
-        relations: ['user'],
-      })
-    )
-    if (!newsletterEmail) {
-      log.info('no newsletter email for', {
-        id: recentEmail.id,
-        to: recentEmail.to,
-        from: recentEmail.from,
-      })
-
-      return {
-        errorCodes: [MarkEmailAsItemErrorCode.NotFound],
-      }
-    }
-
-    const success = await saveNewsletter(
-      {
-        from: recentEmail.from,
-        email: recentEmail.to,
-        title: recentEmail.subject,
-        content: recentEmail.html,
-        url: generateUniqueUrl(),
-        author: parseEmailAddress(recentEmail.from).name,
-        receivedEmailId: recentEmail.id,
+        address: ILike(recentEmail.to),
       },
-      newsletterEmail
-    )
-    if (!success) {
-      log.info('newsletter not created', recentEmail.id)
-
-      return {
-        errorCodes: [MarkEmailAsItemErrorCode.BadRequest],
-      }
-    }
-
-    // update received email type
-    await updateReceivedEmail(recentEmail.id, 'article', uid)
-
-    const text = `A recent email marked as a library item
-                    by: ${uid}
-                    from: ${recentEmail.from}
-                    subject: ${recentEmail.subject}`
-
-    // email us to let us know that an email failed to parse as an article
-    await sendEmail({
-      to: env.sender.feedback,
-      subject: 'A recent email marked as a library item',
-      text,
-      from: env.sender.message,
+      relations: ['user'],
+    })
+  )
+  if (!newsletterEmail) {
+    log.info('no newsletter email for', {
+      id: recentEmail.id,
+      to: recentEmail.to,
+      from: recentEmail.from,
     })
 
     return {
-      success,
+      errorCodes: [MarkEmailAsItemErrorCode.NotFound],
     }
-  } catch (error) {
-    log.error('Error marking email as item', error)
+  }
+
+  const success = await saveNewsletter(
+    {
+      from: recentEmail.from,
+      email: recentEmail.to,
+      title: recentEmail.subject,
+      content: recentEmail.html,
+      url: generateUniqueUrl(),
+      author: parseEmailAddress(recentEmail.from).name || recentEmail.from,
+      receivedEmailId: recentEmail.id,
+    },
+    newsletterEmail
+  )
+  if (!success) {
+    log.info('newsletter not created', recentEmail.id)
 
     return {
       errorCodes: [MarkEmailAsItemErrorCode.BadRequest],
     }
   }
+
+  const text = `A recent email marked as a library item
+                    by: ${uid}
+                    from: ${recentEmail.from}
+                    subject: ${recentEmail.subject}`
+
+  // email us to let us know that an email failed to parse as an article
+  await sendEmail({
+    to: env.sender.feedback,
+    subject: 'A recent email marked as a library item',
+    text,
+    from: env.sender.message,
+  })
+
+  return {
+    success,
+  }
+})
+
+export const replyToEmailResolver = authorized<
+  ReplyToEmailSuccess,
+  ReplyToEmailError,
+  MutationReplyToEmailArgs
+>(async (_, { recentEmailId, reply }, { uid, log }) => {
+  const repo = getRepository(ReceivedEmail)
+  const recentEmail = await repo.findOneBy({
+    id: recentEmailId,
+    user: { id: uid },
+  })
+
+  if (!recentEmail) {
+    log.info('no recent email', recentEmailId)
+
+    return {
+      errorCodes: [ReplyToEmailErrorCode.Unauthorized],
+    }
+  }
+
+  const result = await enqueueSendEmail({
+    to: recentEmail.replyTo || recentEmail.from, // send to the reply-to address if it exists or the from address
+    subject: 'Re: ' + recentEmail.subject,
+    text: reply,
+    from: recentEmail.to,
+  })
+
+  const success = !!result
+
+  if (success) {
+    // update received email reply
+    await repo.update(recentEmailId, { reply })
+  }
+
+  return {
+    success,
+  }
 })

packages/api/src/routers/digest_router.ts

@@ -0,0 +1,242 @@
+import cors from 'cors'
+import express from 'express'
+import { env } from '../env'
+import { TaskState } from '../generated/graphql'
+import { CreateDigestJobSchedule } from '../jobs/ai/create_digest'
+import { getDigest } from '../services/digest'
+import { FeatureName, findGrantedFeatureByName } from '../services/features'
+import { findActiveUser } from '../services/user'
+import { analytics } from '../utils/analytics'
+import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
+import { corsConfig } from '../utils/corsConfig'
+import { enqueueCreateDigest } from '../utils/createTask'
+import { logger } from '../utils/logger'
+import { v4 as uuid } from 'uuid'
+
+interface Feedback {
+  digestRating: number
+  rankingModels: string[]
+  rankingRating: number
+  summaryRating: number
+  summaryModels: string[]
+  voiceRating: number
+  musicRating: number
+  comment?: string
+}
+
+const isFeedback = (data: any): data is Feedback => {
+  return (
+    'digestRating' in data &&
+    'rankingRating' in data &&
+    'summaryRating' in data &&
+    'voiceRating' in data &&
+    'musicRating' in data
+  )
+}
+
+interface CreateDigestRequest {
+  voices?: string[]
+  language?: string
+  rate?: string
+  schedule?: CreateDigestJobSchedule
+  libraryItemIds?: string[]
+}
+
+export function digestRouter() {
+  const router = express.Router()
+
+  // v1 version of create digest api
+  router.post('/v1', cors<express.Request>(corsConfig), async (req, res) => {
+    const token = getTokenByRequest(req)
+
+    let userId: string
+    try {
+      // get claims from token
+      const claims = await getClaimsByToken(token)
+      if (!claims) {
+        logger.info('Token not found')
+        return res.sendStatus(401)
+      }
+
+      // get user by uid from claims
+      userId = claims.uid
+    } catch (error) {
+      logger.info('Error while getting claims from token', error)
+      return res.sendStatus(401)
+    }
+
+    try {
+      const user = await findActiveUser(userId)
+      if (!user) {
+        logger.info(`User not found: ${userId}`)
+        return res.sendStatus(401)
+      }
+
+      const feature = await findGrantedFeatureByName(
+        FeatureName.AIDigest,
+        userId
+      )
+      if (!feature) {
+        logger.info(`${FeatureName.AIDigest} not granted: ${userId}`)
+        return res.sendStatus(403)
+      }
+
+      const data = req.body as CreateDigestRequest
+
+      // check if job is running
+      // if yes then return 202 accepted
+      // else enqueue job
+      const digest = await getDigest(userId)
+      if (digest?.jobState === TaskState.Running) {
+        logger.info(`Digest job is running: ${userId}`)
+        return res.sendStatus(202)
+      }
+
+      // enqueue job and return job id
+      const result = await enqueueCreateDigest(
+        {
+          id: uuid(), // generate job id
+          userId,
+          ...data,
+        },
+        data.schedule
+      )
+
+      // return job id
+      return res.status(201).send(result)
+    } catch (error) {
+      logger.error('Error while enqueuing create digest task', error)
+      return res.sendStatus(500)
+    }
+  })
+
+  // v1 version of get digest api
+  router.get('/v1', cors<express.Request>(corsConfig), async (req, res) => {
+    const token = getTokenByRequest(req)
+
+    let userId: string
+    try {
+      // get claims from token
+      const claims = await getClaimsByToken(token)
+      if (!claims) {
+        logger.info('Token not found')
+        return res.sendStatus(401)
+      }
+
+      // get user by uid from claims
+      userId = claims.uid
+    } catch (error) {
+      logger.info('Error while getting claims from token', error)
+      return res.sendStatus(401)
+    }
+
+    try {
+      const user = await findActiveUser(userId)
+      if (!user) {
+        logger.info(`User not found: ${userId}`)
+        return res.sendStatus(401)
+      }
+
+      const feature = await findGrantedFeatureByName(
+        FeatureName.AIDigest,
+        userId
+      )
+      if (!feature) {
+        logger.info(`${FeatureName.AIDigest} not granted: ${userId}`)
+        return res.sendStatus(403)
+      }
+
+      // get the digest from redis
+      const digest = await getDigest(userId)
+      if (!digest) {
+        logger.info(`Digest not found: ${userId}`)
+        return res.sendStatus(404)
+      }
+
+      if (digest.jobState === TaskState.Running) {
+        // if job is running then return job state
+        return res.send({
+          jobId: digest.id,
+          jobState: digest.jobState,
+        })
+      }
+
+      // if job is done then return the digest
+      return res.send(digest)
+    } catch (error) {
+      logger.error('Error while getting digest', error)
+      return res.sendStatus(500)
+    }
+  })
+
+  // v1 version of sending feedback api
+  router.post(
+    '/v1/feedback',
+    cors<express.Request>(corsConfig),
+    async (req, res) => {
+      const token = getTokenByRequest(req)
+
+      let userId: string
+      try {
+        // get claims from token
+        const claims = await getClaimsByToken(token)
+        if (!claims) {
+          logger.info('Token not found')
+          return res.sendStatus(401)
+        }
+
+        // get user by uid from claims
+        userId = claims.uid
+      } catch (error) {
+        logger.info('Error while getting claims from token', error)
+        return res.sendStatus(401)
+      }
+
+      try {
+        const user = await findActiveUser(userId)
+        if (!user) {
+          logger.info(`User not found: ${userId}`)
+          return res.sendStatus(401)
+        }
+
+        const feature = await findGrantedFeatureByName(
+          FeatureName.AIDigest,
+          userId
+        )
+        if (!feature) {
+          logger.info(`${FeatureName.AIDigest} not granted: ${userId}`)
+          return res.sendStatus(403)
+        }
+
+        // get feedback from request body
+        if (!isFeedback(req.body)) {
+          logger.info('Invalid feedback format')
+          return res.sendStatus(400)
+        }
+
+        const feedback = req.body
+        logger.info(`Sending feedback: ${JSON.stringify(feedback)}`)
+
+        // remove comment from feedback before sending to analytics
+        delete feedback.comment
+        // send feedback to analytics
+        analytics.capture({
+          distinctId: userId,
+          event: 'digest_feedback',
+          properties: {
+            ...feedback,
+            env: env.server.apiEnv,
+          },
+        })
+
+        // return success
+        return res.sendStatus(200)
+      } catch (error) {
+        logger.error('Error while saving feedback', error)
+        return res.sendStatus(500)
+      }
+    }
+  )
+
+  return router
+}

packages/api/src/routers/svc/emails.ts

@@ -27,6 +27,7 @@ interface EmailMessage {
   text: string
   forwardedFrom?: string
   receivedEmailId: string
+  replyTo?: string
 }
 
 function isEmailMessage(data: any): data is EmailMessage {
@@ -82,7 +83,7 @@ export function emailsServiceRouter() {
         const savedNewsletter = await saveNewsletter(
           {
             title: getTitleFromEmailSubject(data.subject),
-            author: parsedFrom.name,
+            author: parsedFrom.name || data.from,
             url: generateUniqueUrl(),
             content: data.html || data.text,
             receivedEmailId: data.receivedEmailId,
@@ -165,7 +166,9 @@ export function emailsServiceRouter() {
         req.body.subject,
         req.body.text,
         req.body.html,
-        user.id
+        user.id,
+        'non-article',
+        req.body.replyTo
       )
 
       analytics.capture({

packages/api/src/schema.ts

@@ -2547,6 +2547,8 @@ const schema = gql`
     type: String!
     text: String!
     html: String
+    replyTo: String
+    reply: String
     createdAt: Date!
   }
 
@@ -3066,6 +3068,27 @@ const schema = gql`
     FAILED_TO_CREATE_TASK
   }
 
+  union ReplyToEmailResult = ReplyToEmailSuccess | ReplyToEmailError
+
+  type ReplyToEmailSuccess {
+    success: Boolean!
+  }
+
+  type ReplyToEmailError {
+    errorCodes: [ReplyToEmailErrorCode!]!
+  }
+
+  enum ReplyToEmailErrorCode {
+    UNAUTHORIZED
+  }
+
+  enum AllowedReply {
+    YES
+    OKAY
+    CONFIRM
+    SUBSCRIBE
+  }
+
   # Mutations
   type Mutation {
     googleLogin(input: GoogleLoginInput!): LoginResult!
@@ -3165,6 +3188,7 @@ const schema = gql`
       contentType: String!
     ): UploadImportFileResult!
     markEmailAsItem(recentEmailId: ID!): MarkEmailAsItemResult!
+    replyToEmail(recentEmailId: ID!, reply: AllowedReply!): ReplyToEmailResult!
     bulkAction(
       query: String!
       action: BulkActionType!

packages/api/src/server.ts

@@ -21,6 +21,7 @@ import { aiSummariesRouter } from './routers/ai_summary_router'
 import { articleRouter } from './routers/article_router'
 import { authRouter } from './routers/auth/auth_router'
 import { mobileAuthRouter } from './routers/auth/mobile/mobile_auth_router'
+import { digestRouter } from './routers/digest_router'
 import { integrationRouter } from './routers/integration_router'
 import { localDebugRouter } from './routers/local_debug_router'
 import { notificationRouter } from './routers/notification_router'
@@ -47,11 +48,7 @@ import { apiLimiter, authLimiter } from './utils/rate_limit'
 
 const PORT = process.env.PORT || 4000
 
-export const createApp = (): {
-  app: Express
-  apollo: ApolloServer
-  httpServer: Server
-} => {
+export const createApp = (): Express => {
   const app = express()
 
   // eslint-disable-next-line @typescript-eslint/ban-ts-comment
@@ -93,6 +90,7 @@ export const createApp = (): {
   app.use('/api/notification', notificationRouter())
   app.use('/api/integration', integrationRouter())
   app.use('/api/tasks', taskRouter())
+  app.use('/api/digest', digestRouter())
   app.use('/svc/pubsub/content', contentServiceRouter())
   app.use('/svc/pubsub/links', linkServiceRouter())
   app.use('/svc/pubsub/newsletters', newsletterServiceRouter())
@@ -136,10 +134,7 @@ export const createApp = (): {
     res.end(await prom.register.metrics())
   })
 
-  const apollo = makeApolloServer(app)
-  const httpServer = createServer(app)
-
-  return { app, apollo, httpServer }
+  return app
 }
 
 const main = async (): Promise<void> => {
@@ -154,19 +149,17 @@ const main = async (): Promise<void> => {
     await redisDataSource.initialize()
   }
 
-  const { app, apollo, httpServer } = createApp()
-
+  const app = createApp()
+  const apollo = makeApolloServer(app)
   await apollo.start()
   apollo.applyMiddleware({ app, path: '/api/graphql', cors: corsConfig })
 
-  if (!env.dev.isLocal) {
-    const mwLogger = loggers.get('express', { levels: config.syslog.levels })
-    const transport = buildLoggerTransport('express')
-    const mw = await lw.express.makeMiddleware(mwLogger, transport)
-    app.use(mw)
-  }
+  const mwLogger = loggers.get('express', { levels: config.syslog.levels })
+  const transport = buildLoggerTransport('express')
+  const mw = await lw.express.makeMiddleware(mwLogger, transport)
+  app.use(mw)
 
-  const listener = httpServer.listen({ port: PORT }, async () => {
+  const listener = app.listen({ port: PORT }, async () => {
     const logger = buildLogger('app.dispatch')
     logger.notice(`🚀 Server ready at ${apollo.graphqlPath}`)
   })
@@ -181,15 +174,14 @@ const main = async (): Promise<void> => {
   listener.timeout = 640 * 1000 // match headersTimeout
 
   const gracefulShutdown = async (signal: string) => {
+    console.log(`[api]: Received ${signal}, closing server...`)
+    await apollo.stop()
+    console.log('[api]: Apollo server stopped')
+
     console.log('[posthog]: flushing events')
     await analytics.shutdownAsync()
     console.log('[posthog]: events flushed')
 
-    console.log(`[api]: Received ${signal}, closing server...`)
-
-    await apollo.stop()
-    console.log('[api]: Apollo server stopped')
-
     await new Promise<void>((resolve) => {
       listener.close((err) => {
         console.log('[api]: Express listener closed')

packages/api/src/services/digest.ts

@@ -0,0 +1,51 @@
+import { redisDataSource } from '../redis_data_source'
+import { SpeechFile } from '@omnivore/text-to-speech-handler'
+import { logger } from '../utils/logger'
+import { TaskState } from '../generated/graphql'
+
+interface Chapter {
+  title: string
+  id: string
+  url: string
+  wordCount: number
+  thumbnail?: string
+}
+
+export interface Digest {
+  id: string
+  jobState: TaskState
+
+  createdAt?: Date
+  description?: string
+  byline?: string
+  url?: string
+  title?: string
+  content?: string
+  chapters?: Chapter[]
+
+  urlsToAudio?: string[]
+  speechFiles?: SpeechFile[]
+}
+
+const digestKey = (userId: string) => `digest:${userId}`
+
+export const getDigest = async (userId: string): Promise<Digest | null> => {
+  const digest = await redisDataSource.redisClient?.get(digestKey(userId))
+  return digest ? (JSON.parse(digest) as Digest) : null
+}
+
+export const writeDigest = async (userId: string, digest: Digest) => {
+  // write to redis
+  const result = await redisDataSource.redisClient?.set(
+    digestKey(userId),
+    JSON.stringify(digest),
+    'EX',
+    60 * 60 * 24 * 7 // 1 week
+  )
+
+  if (!result) {
+    const msg = `Error while writing digest to redis: ${userId}`
+    logger.error(msg)
+    throw new Error(msg)
+  }
+}

packages/api/src/services/features.ts

@@ -9,12 +9,14 @@ import { logger } from '../utils/logger'
 const MAX_ULTRA_REALISTIC_USERS = 1500
 const MAX_YOUTUBE_TRANSCRIPT_USERS = 500
 const MAX_NOTION_USERS = 1000
+const MAX_AIDIGEST_USERS = 5
 
 export enum FeatureName {
   AISummaries = 'ai-summaries',
   YouTubeTranscripts = 'youtube-transcripts',
   UltraRealisticVoice = 'ultra-realistic-voice',
   Notion = 'notion',
+  AIDigest = 'ai-digest',
 }
 
 export const getFeatureName = (name: string): FeatureName | undefined => {
@@ -40,6 +42,8 @@ export const optInFeature = async (
       )
     case FeatureName.Notion:
       return optInLimitedFeature(FeatureName.Notion, uid, MAX_NOTION_USERS)
+    case FeatureName.AIDigest:
+      return optInLimitedFeature(FeatureName.AIDigest, uid, MAX_AIDIGEST_USERS)
     default:
       return undefined
   }

packages/api/src/services/library_item.ts

@@ -626,12 +626,10 @@ export const buildQuery = (
 ) => {
   // select all columns except content
   const selects: Select[] = getColumns(libraryItemRepository)
-    .map((column) => ({ column: `library_item.${column}` }))
     .filter(
-      (select) =>
-        select.column !== 'library_item.readableContent' &&
-        select.column !== 'library_item.originalContent'
+      (select) => select !== 'readableContent' && select !== 'originalContent'
     )
+    .map((column) => ({ column: `library_item.${column}` }))
 
   const parameters: ObjectLiteral[] = []
   const orders: Sort[] = []
@@ -652,10 +650,18 @@ export const buildQuery = (
   queryBuilder.where('library_item.user_id = :userId', { userId })
 
   // add select
-  selects.forEach((select) => {
+  selects.forEach((select, index) => {
+    if (index === 0) {
+      queryBuilder.select(select.column, select.alias)
+    }
+
     queryBuilder.addSelect(select.column, select.alias)
   })
 
+  if (args.includeContent) {
+    queryBuilder.addSelect('library_item.readableContent')
+  }
+
   if (!args.includePending) {
     queryBuilder.andWhere("library_item.state <> 'PROCESSING'")
   }
@@ -755,17 +761,13 @@ export const findRecentLibraryItems = async (
 
 export const findLibraryItemsByIds = async (ids: string[], userId: string) => {
   const selectColumns = getColumns(libraryItemRepository)
-    .filter(
-      (column) => column !== 'readableContent' && column !== 'originalContent'
-    )
+    .filter((column) => column !== 'originalContent')
     .map((column) => `library_item.${column}`)
   return authTrx(
     async (tx) =>
       tx
         .createQueryBuilder(LibraryItem, 'library_item')
         .select(selectColumns)
-        .leftJoinAndSelect('library_item.labels', 'labels')
-        .leftJoinAndSelect('library_item.highlights', 'highlights')
         .where('library_item.id IN (:...ids)', { ids })
         .getMany(),
     undefined,

packages/api/src/services/received_emails.ts

@@ -8,7 +8,8 @@ export const saveReceivedEmail = async (
   text: string,
   html: string,
   userId: string,
-  type: 'article' | 'non-article' = 'non-article'
+  type: 'article' | 'non-article' = 'non-article',
+  replyTo?: string
 ): Promise<ReceivedEmail> => {
   return authTrx(
     (t) =>
@@ -20,6 +21,7 @@ export const saveReceivedEmail = async (
         html,
         type,
         user: { id: userId },
+        replyTo,
       }),
     undefined,
     userId

packages/api/src/services/save_newsletter_email.ts

@@ -76,59 +76,5 @@ export const saveNewsletter = async (
     return false
   }
 
-  // sends push notification
-  // const deviceTokens = await getDeviceTokensByUserId(newsletterEmail.user.id)
-  // if (!deviceTokens) {
-  //   logger.info('Device tokens not set:', newsletterEmail.user.id)
-  //   return true
-  // }
-
-  // const multicastMessage = messageForLink(page, deviceTokens)
-  // await sendMulticastPushNotifications(
-  //   newsletterEmail.user.id,
-  //   multicastMessage,
-  //   'newsletter'
-  // )
-
   return true
 }
-
-// const messageForLink = (
-//   link: Page,
-//   deviceTokens: UserDeviceToken[]
-// ): MulticastMessage => {
-//   let title = '📫 - An article was added to your Omnivore Inbox'
-
-//   if (link.author) {
-//     title = `📫 - ${link.author} has published a new article`
-//   }
-
-//   const pushData = !link
-//     ? undefined
-//     : {
-//         link: Buffer.from(
-//           JSON.stringify({
-//             id: link.id,
-//             url: link.url,
-//             slug: link.slug,
-//             title: link.title,
-//             image: link.image,
-//             author: link.author,
-//             isArchived: !!link.archivedAt,
-//             contentReader: ContentReader.Web,
-//             readingProgressPercent: link.readingProgressPercent,
-//             readingProgressAnchorIndex: link.readingProgressAnchorIndex,
-//           })
-//         ).toString('base64'),
-//       }
-
-//   return {
-//     notification: {
-//       title: title,
-//       body: link.title,
-//       imageUrl: link.image || undefined,
-//     },
-//     data: pushData,
-//     tokens: deviceTokens.map((token) => token.token),
-//   }
-// }

packages/api/src/services/send_emails.ts

@@ -18,7 +18,7 @@ export const sendNewAccountVerificationEmail = async (user: {
   }
 
   const result = await enqueueSendEmail({
-    emailAddress: user.email,
+    to: user.email,
     dynamicTemplateData: dynamicTemplateData,
     templateId: env.sendgrid.confirmationTemplateId,
   })
@@ -78,7 +78,7 @@ export const sendAccountChangeEmail = async (user: {
   }
 
   const result = await enqueueSendEmail({
-    emailAddress: user.email,
+    to: user.email,
     dynamicTemplateData: dynamicTemplateData,
     templateId: env.sendgrid.verificationTemplateId,
   })
@@ -100,7 +100,7 @@ export const sendPasswordResetEmail = async (user: {
   }
 
   const result = await enqueueSendEmail({
-    emailAddress: user.email,
+    to: user.email,
     dynamicTemplateData: dynamicTemplateData,
     templateId: env.sendgrid.resetPasswordTemplateId,
   })

packages/api/src/services/service_usage.ts

@@ -0,0 +1,31 @@
+import { Between } from 'typeorm'
+import { ServiceUsage } from '../entity/service_usage'
+import { authTrx, getRepository } from '../repository'
+import { DateTime } from 'luxon'
+
+const repo = getRepository(ServiceUsage)
+
+export const countDailyServiceUsage = async (
+  userId: string,
+  action: string
+) => {
+  return authTrx((tx) =>
+    tx.withRepository(repo).countBy({
+      user: { id: userId },
+      action,
+      createdAt: Between(
+        DateTime.now().startOf('day').toJSDate(),
+        DateTime.now().endOf('day').toJSDate()
+      ),
+    })
+  )
+}
+
+export const createServiceUsage = async (userId: string, action: string) => {
+  return authTrx((tx) =>
+    tx.withRepository(repo).save({
+      user: { id: userId },
+      action,
+    })
+  )
+}

packages/api/src/services/update_pdf_content.ts

@@ -8,16 +8,17 @@ import { findUploadFileById, setFileUploadComplete } from './upload_file'
 
 export interface UpdateContentMessage {
   fileId: string
-  content: string
+  content?: string
   title?: string
   author?: string
   description?: string
+  state?: LibraryItemState
 }
 
 export const isUpdateContentMessage = (
   data: any
 ): data is UpdateContentMessage => {
-  return 'fileId' in data && 'content' in data
+  return 'fileId' in data
 }
 
 export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
@@ -51,16 +52,16 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
   }
 
   const itemToUpdate: QueryDeepPartialEntity<LibraryItem> = {
-    originalContent: msg.content,
+    title: msg.title,
+    description: msg.description,
+    author: msg.author,
+    // content may not be present if we failed to parse the file
+    readableContent: msg.content,
+    // This event is fired after the file is fully uploaded,
+    // so along with updating content, we mark it as
+    // succeeded or failed based on the message state
+    state: msg.state || LibraryItemState.Succeeded,
   }
-  if (msg.title) itemToUpdate.title = msg.title
-  if (msg.author) itemToUpdate.author = msg.author
-  if (msg.description) itemToUpdate.description = msg.description
-
-  // This event is fired after the file is fully uploaded,
-  // so along with updating content, we mark it as
-  // succeeded.
-  itemToUpdate.state = LibraryItemState.Succeeded
 
   try {
     const uploadFileData = await setFileUploadComplete(
@@ -80,7 +81,8 @@ export const updateContentForFileItem = async (msg: UpdateContentMessage) => {
   logger.info('Updating library item text', {
     id: libraryItem.id,
     result,
-    content: msg.content.substring(0, 20),
+    content: msg.content?.substring(0, 20),
+    state: msg.state,
   })
 
   return true

packages/api/src/services/update_subscription.ts

@@ -49,7 +49,7 @@ export const updateSubscription = async (
     lastFetchedChecksum: newData.lastFetchedChecksum || undefined,
     status: newData.status || undefined,
     scheduledAt: newData.scheduledAt || undefined,
-    failedAt: newData.failedAt || undefined,
+    failedAt: newData.failedAt,
     autoAddToLibrary: newData.autoAddToLibrary ?? undefined,
     isPrivate: newData.isPrivate ?? undefined,
     fetchContentType: newData.fetchContentType || undefined,

packages/api/src/services/upload_file.ts

@@ -18,6 +18,7 @@ import {
 } from '../utils/uploads'
 import { validateUrl } from './create_page_save_request'
 import { createOrUpdateLibraryItem } from './library_item'
+import { v4 as uuid } from 'uuid'
 
 const isFileUrl = (url: string): boolean => {
   const parsedUrl = new URL(url)
@@ -90,8 +91,18 @@ export const uploadFile = async (
     }
   }
 
+  let url = input.url
+
+  const uploadFileId = uuid()
+  const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName)
+  // If this is a file URL, we swap in a special URL
+  if (isFileUrl(url)) {
+    url = `https://omnivore.app/attachments/${uploadFilePathName}`
+  }
+
   const uploadFileData = await authTrx((t) =>
     t.getRepository(UploadFile).save({
+      id: uploadFileId,
       url: input.url,
       user: { id: uid },
       fileName,
@@ -99,24 +110,11 @@ export const uploadFile = async (
       contentType: input.contentType,
     })
   )
-  const uploadFileId = uploadFileData.id
-  const uploadFilePathName = generateUploadFilePathName(uploadFileId, fileName)
   const uploadSignedUrl = await generateUploadSignedUrl(
     uploadFilePathName,
     input.contentType
   )
 
-  // If this is a file URL, we swap in a special URL
-  const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}`
-  if (isFileUrl(input.url)) {
-    await authTrx(async (tx) => {
-      await tx.getRepository(UploadFile).update(uploadFileId, {
-        url: attachmentUrl,
-        status: UploadFileStatus.Initialized,
-      })
-    })
-  }
-
   const itemType = itemTypeForContentType(input.contentType)
   if (input.createPageEntry) {
     // If we have a file:// URL, don't try to match it
@@ -125,7 +123,7 @@ export const uploadFile = async (
     const item = await createOrUpdateLibraryItem(
       {
         id: input.clientRequestId || undefined,
-        originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url,
+        originalUrl: url,
         user: { id: uid },
         title,
         readableContent: '',

packages/api/src/utils/auth.ts

@@ -124,8 +124,7 @@ export const getTokenByRequest = (req: express.Request): string | undefined => {
   return (
     req.header(OmnivoreAuthorizationHeader) ||
     req.headers.authorization ||
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
-    (req.cookies?.auth as string)
+    (req.cookies.auth as string | undefined)
   )
 }
 

packages/api/src/utils/createTask.ts

@@ -14,8 +14,15 @@ import { env } from '../env'
 import {
   ArticleSavingRequestStatus,
   CreateLabelInput,
+  TaskState,
 } from '../generated/graphql'
 import { AISummarizeJobData, AI_SUMMARIZE_JOB_NAME } from '../jobs/ai-summarize'
+import {
+  CreateDigestJobData,
+  CreateDigestJobResponse,
+  CreateDigestJobSchedule,
+  CREATE_DIGEST_JOB,
+} from '../jobs/ai/create_digest'
 import { BulkActionData, BULK_ACTION_JOB_NAME } from '../jobs/bulk_action'
 import { CallWebhookJobData, CALL_WEBHOOK_JOB_NAME } from '../jobs/call_webhook'
 import { THUMBNAIL_JOB } from '../jobs/find_thumbnail'
@@ -35,7 +42,7 @@ import {
   REFRESH_ALL_FEEDS_JOB_NAME,
   REFRESH_FEED_JOB_NAME,
 } from '../jobs/rss/refreshAllFeeds'
-import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/send_email'
+import { SendEmailJobData, SEND_EMAIL_JOB } from '../jobs/email/send_email'
 import { SYNC_READ_POSITIONS_JOB_NAME } from '../jobs/sync_read_positions'
 import { TriggerRuleJobData, TRIGGER_RULE_JOB_NAME } from '../jobs/trigger_rule'
 import {
@@ -52,6 +59,7 @@ import { CreateTaskError } from './errors'
 import { stringToHash } from './helpers'
 import { logger } from './logger'
 import View = google.cloud.tasks.v2.Task.View
+import { writeDigest } from '../services/digest'
 
 // Instantiates a client.
 const client = new CloudTasksClient()
@@ -81,9 +89,9 @@ export const getJobPriority = (jobName: string): number => {
     case `${REFRESH_FEED_JOB_NAME}_high`:
       return 10
     case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
-      return 20
     case `${REFRESH_FEED_JOB_NAME}_low`:
     case EXPORT_ITEM_JOB_NAME:
+    case CREATE_DIGEST_JOB:
       return 50
     case EXPORT_ALL_ITEMS_JOB_NAME:
     case REFRESH_ALL_FEEDS_JOB_NAME:
@@ -853,4 +861,44 @@ export const enqueueSendEmail = async (jobData: SendEmailJobData) => {
   })
 }
 
+export const enqueueCreateDigest = async (
+  data: CreateDigestJobData,
+  schedule?: CreateDigestJobSchedule
+): Promise<CreateDigestJobResponse> => {
+  const queue = await getBackendQueue()
+  if (!queue) {
+    throw new Error('No queue found')
+  }
+
+  const job = await queue.add(CREATE_DIGEST_JOB, data, {
+    jobId: data.id, // dedupe by job id
+    removeOnComplete: true,
+    removeOnFail: true,
+    attempts: 3,
+    priority: getJobPriority(CREATE_DIGEST_JOB),
+    repeat: schedule
+      ? {
+          immediately: true, // run immediately
+          pattern: schedule === 'daily' ? '0 13 * * *' : '0 13 * * 7', // every day or every Sunday at 1PM
+          utc: true,
+        }
+      : undefined,
+  })
+
+  logger.info('create digest job enqueued', { jobId: job.id })
+
+  const digest = {
+    id: data.id,
+    jobState: TaskState.Running,
+  }
+
+  // update digest job state in redis
+  await writeDigest(data.userId, digest)
+
+  return {
+    jobId: digest.id,
+    jobState: digest.jobState,
+  }
+}
+
 export default createHttpTaskWithToken

packages/api/src/utils/parser.ts

@@ -703,6 +703,10 @@ export const htmlToMarkdown = (html: string) => {
   return nhm.translate(/* html */ html)
 }
 
+export const markdownToHtml = (markdown: string) => {
+  return nhm.translate(/* markdown */ markdown)
+}
+
 export const getDistillerResult = async (
   uid: string,
   html: string

packages/api/test/global-teardown.ts

@@ -7,16 +7,16 @@ export const mochaGlobalTeardown = async () => {
   await stopApolloServer()
   console.log('apollo server stopped')
 
-  await appDataSource.destroy()
-  console.log('db connection closed')
-
   if (env.redis.cache.url) {
-    await redisDataSource.shutdown()
-    console.log('redis connection closed')
-
     if (redisDataSource.workerRedisClient) {
       await stopWorker()
       console.log('worker closed')
     }
+
+    await redisDataSource.shutdown()
+    console.log('redis connection closed')
   }
+
+  await appDataSource.destroy()
+  console.log('db connection closed')
 }

packages/api/test/resolvers/highlight.test.ts

@@ -1,10 +1,9 @@
+/* eslint-disable @typescript-eslint/restrict-template-expressions */
 import * as chai from 'chai'
 import { expect } from 'chai'
 import chaiString from 'chai-string'
 import 'mocha'
-import { Highlight } from '../../src/entity/highlight'
 import { User } from '../../src/entity/user'
-import { getRepository } from '../../src/repository'
 import {
   createHighlight,
   deleteHighlightById,
@@ -157,7 +156,7 @@ describe('Highlights API', () => {
       .post('/local/debug/fake-user-login')
       .send({ fakeEmail: user.email })
 
-    authToken = res.body.authToken
+    authToken = res.body.authToken as string
     itemId = (await createTestLibraryItem(user.id)).id
   })
 

packages/api/test/resolvers/labels.test.ts

@@ -145,7 +145,7 @@ describe('Labels API', () => {
       })
     })
 
-    context('when name exists', () => {
+    context('when name exists in the user library', () => {
       let existingLabel: Label
 
       before(async () => {
@@ -177,6 +177,32 @@ describe('Labels API', () => {
       })
     })
 
+    context('when name exists in the other user library', () => {
+      let existingLabel: Label
+      let otherUser: User
+
+      before(async () => {
+        otherUser = await createTestUser('otherUser')
+        existingLabel = await createLabel('label3', '#ffffff', otherUser.id)
+      })
+
+      after(async () => {
+        // delete other user will also delete the label
+        await deleteUser(otherUser.id)
+      })
+
+      it('creates the label', async () => {
+        const res = await graphqlRequest(query, authToken, {
+          input: { name: existingLabel.name },
+        }).expect(200)
+        const label = await findLabelById(
+          res.body.data.createLabel.label.id,
+          user.id
+        )
+        expect(label).to.exist
+      })
+    })
+
     it('responds status code 400 when invalid query', async () => {
       const invalidQuery = `
         mutation {

packages/api/test/util.ts

@@ -2,11 +2,13 @@ import { ConnectionOptions, Job, QueueEvents, Worker } from 'bullmq'
 import { nanoid } from 'nanoid'
 import supertest from 'supertest'
 import { v4 } from 'uuid'
+import { makeApolloServer } from '../src/apollo'
 import { createWorker, QUEUE_NAME } from '../src/queue-processor'
 import { createApp } from '../src/server'
 import { corsConfig } from '../src/utils/corsConfig'
 
-const { app, apollo } = createApp()
+const app = createApp()
+const apollo = makeApolloServer(app)
 export const request = supertest(app)
 let worker: Worker
 let queueEvents: QueueEvents

packages/db/migrations/0172.do.service_usage.sql

@@ -0,0 +1,39 @@
+-- Type: DO
+-- Name: service_usage
+-- Description: Create table for tracking service usage and enforce limit
+
+BEGIN;
+
+ALTER TABLE omnivore.received_emails
+    ADD COLUMN reply_to TEXT,
+    ADD COLUMN reply TEXT;
+
+CREATE TABLE omnivore.subscription_plan (
+    id SERIAL PRIMARY KEY,
+    name VARCHAR(255) NOT NULL,
+    description TEXT,
+    max_emails_sent_per_day INT NOT NULL,
+    created_at timestamptz NOT NULL default current_timestamp
+);
+
+INSERT INTO omnivore.subscription_plan (id, name, description, max_emails_sent_per_day)
+VALUES (1, 'Basic', 'Basic plan', 3);
+
+CREATE TABLE omnivore.service_usage (
+	id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    user_id uuid NOT NULL REFERENCES omnivore.user,
+    action VARCHAR(255) NOT NULL,
+    created_at timestamptz NOT NULL default current_timestamp
+);
+
+CREATE INDEX ON omnivore.service_usage (user_id);
+
+ALTER TABLE omnivore.service_usage ENABLE ROW LEVEL SECURITY;
+
+CREATE POLICY service_usage_policy on omnivore.service_usage
+    USING (user_id = omnivore.get_current_user_id())
+    WITH CHECK (user_id = omnivore.get_current_user_id());
+
+GRANT SELECT, INSERT ON omnivore.service_usage TO omnivore_user;
+
+COMMIT;

packages/db/migrations/0172.undo.service_usage.sql

@@ -0,0 +1,15 @@
+-- Type: UNDO
+-- Name: service_usage
+-- Description: Create table for tracking service usage and enforce limit
+
+BEGIN;
+
+DROP TABLE IF EXISTS omnivore.service_usage;
+
+DROP TABLE IF EXISTS omnivore.subscription_plan;
+
+ATLER TABLE omnivore.received_emails
+    DROP COLUMN IF EXISTS reply_to,
+    DROP COLUMN IF EXISTS reply;
+
+COMMIT;

packages/db/migrations/0173.do.alter_labels_table_policy.sql

@@ -0,0 +1,11 @@
+-- Type: DO
+-- Name: alter_labels_table_policy
+-- Description: Alter labels table select policy to check user_id
+
+BEGIN;
+
+ALTER POLICY read_labels ON omnivore.labels
+    TO omnivore_user
+    USING (user_id = omnivore.get_current_user_id());
+
+COMMIT;

packages/db/migrations/0173.undo.alter_labels_table_policy.sql

@@ -0,0 +1,11 @@
+-- Type: UNDO
+-- Name: alter_labels_table_policy
+-- Description: Alter labels table select policy to check user_id
+
+BEGIN;
+
+ALTER POLICY read_labels ON omnivore.labels
+    TO omnivore_user
+    USING (true);
+
+COMMIT;

packages/inbound-email-handler/.eslintrc

@@ -2,5 +2,13 @@
   "extends": "../../.eslintrc",
   "parserOptions": {
     "project": "tsconfig.json"
+  },
+  "rules": {
+    "@typescript-eslint/no-floating-promises": [
+      "error",
+      {
+        "ignoreIIFE": true
+      }
+    ]
   }
-}
+}

packages/inbound-email-handler/mocha-config.json

@@ -1,4 +1,6 @@
 {
     "extension": ["ts"],
-    "spec": "test/**/*.test.ts"
+    "spec": "test/**/*.test.ts",
+    "require": ["test/global-teardown.ts"],
+    "timeout": 10000
   }

packages/inbound-email-handler/package.json

@@ -22,7 +22,9 @@
   },
   "devDependencies": {
     "@types/addressparser": "^1.0.1",
+    "@types/chai": "^4.3.6",
     "@types/json-bigint": "^1.0.1",
+    "@types/mocha": "^10.0.0",
     "@types/node": "^14.11.2",
     "@types/rfc2047": "^2.0.1",
     "@types/showdown": "^2.0.1",
@@ -32,17 +34,16 @@
   },
   "dependencies": {
     "@google-cloud/functions-framework": "3.1.2",
-    "@google-cloud/pubsub": "^4.0.0",
-    "@omnivore/content-handler": "1.0.0",
-    "@sendgrid/client": "^7.6.0",
+    "@google-cloud/storage": "^7.0.1",
     "@sentry/serverless": "^7.77.0",
     "addressparser": "^1.0.1",
-    "axios": "^0.27.2",
-    "jsonwebtoken": "^8.5.1",
+    "bullmq": "^5.1.1",
+    "dotenv": "^8.2.0",
+    "ioredis": "^5.3.2",
     "parse-headers": "^2.0.4",
     "parse-multipart-data": "^1.2.1",
     "rfc2047": "^4.0.1",
-    "showdown": "^2.1.0"
+    "uuid": "^8.3.1"
   },
   "volta": {
     "extends": "../../package.json"

packages/inbound-email-handler/src/attachment.ts

@@ -1,8 +1,11 @@
-import axios, { AxiosResponse } from 'axios'
-import * as jwt from 'jsonwebtoken'
-import { promisify } from 'util'
+import { Storage } from '@google-cloud/storage'
+import { v4 as uuid } from 'uuid'
+import { EmailJobType, queueEmailJob } from './job'
 
-const signToken = promisify(jwt.sign)
+const storage = process.env.GCS_UPLOAD_SA_KEY_FILE_PATH
+  ? new Storage({ keyFilename: process.env.GCS_UPLOAD_SA_KEY_FILE_PATH })
+  : new Storage()
+const bucketName = process.env.GCS_UPLOAD_BUCKET || 'omnivore-files'
 
 export interface Attachment {
   contentType: string
@@ -10,11 +13,6 @@ export interface Attachment {
   filename: string | undefined
 }
 
-type UploadResponse = {
-  id: string
-  url: string
-}
-
 export const isAttachment = (contentType: string, data: Buffer): boolean => {
   return (
     (contentType === 'application/pdf' ||
@@ -23,11 +21,26 @@ export const isAttachment = (contentType: string, data: Buffer): boolean => {
   )
 }
 
+export const uploadToBucket = async (
+  fileName: string,
+  data: Buffer,
+  options?: { contentType?: string; public?: boolean }
+) => {
+  const uploadFileId = uuid()
+
+  await storage
+    .bucket(bucketName)
+    .file(`u/${uploadFileId}/${fileName}`)
+    .save(data, { ...options, timeout: 30000 })
+
+  return uploadFileId
+}
+
 export const handleAttachments = async (
-  email: string,
+  from: string,
+  to: string,
   subject: string,
-  attachments: Attachment[],
-  receivedEmailId: string
+  attachments: Attachment[]
 ): Promise<void> => {
   for await (const attachment of attachments) {
     const { contentType, data } = attachment
@@ -36,98 +49,20 @@ export const handleAttachments = async (
         ? 'attachment.pdf'
         : 'attachment.epub'
 
-    try {
-      const uploadResult = await getUploadIdAndSignedUrl(
-        email,
-        filename,
-        contentType
-      )
-      if (!uploadResult.url || !uploadResult.id) {
-        console.log('failed to create upload request', uploadResult)
-        return
-      }
-      await uploadToSignedUrl(uploadResult.url, data, contentType)
-      await createArticle(email, uploadResult.id, subject, receivedEmailId)
-    } catch (error) {
-      console.error('handleAttachments error', error)
-    }
-  }
-}
+    const uploadFileId = await uploadToBucket(filename, data, {
+      contentType,
+      public: false,
+    })
 
-const getUploadIdAndSignedUrl = async (
-  email: string,
-  fileName: string,
-  contentType: string
-): Promise<UploadResponse> => {
-  if (process.env.JWT_SECRET === undefined) {
-    throw new Error('JWT_SECRET is not defined')
-  }
-  const auth = await signToken(email, process.env.JWT_SECRET)
-  const data = {
-    fileName,
-    email,
-    contentType,
-  }
-
-  if (process.env.INTERNAL_SVC_ENDPOINT === undefined) {
-    throw new Error('REST_BACKEND_ENDPOINT is not defined')
-  }
-  const response = await axios.post(
-    `${process.env.INTERNAL_SVC_ENDPOINT}svc/email-attachment/upload`,
-    data,
-    {
-      headers: {
-        Authorization: `${auth as string}`,
-        'Content-Type': 'application/json',
+    await queueEmailJob(EmailJobType.SaveAttachment, {
+      from,
+      to,
+      uploadFile: {
+        fileName: filename,
+        contentType,
+        id: uploadFileId,
       },
-    }
-  )
-  return response.data as UploadResponse
-}
-
-const uploadToSignedUrl = async (
-  uploadUrl: string,
-  data: Buffer,
-  contentType: string
-): Promise<AxiosResponse> => {
-  return axios.put(uploadUrl, data, {
-    headers: {
-      'Content-Type': contentType,
-    },
-    maxBodyLength: 1000000000,
-    maxContentLength: 100000000,
-  })
-}
-
-const createArticle = async (
-  email: string,
-  uploadFileId: string,
-  subject: string,
-  receivedEmailId: string
-): Promise<AxiosResponse> => {
-  const data = {
-    email,
-    uploadFileId,
-    subject,
-    receivedEmailId,
+      subject,
+    })
   }
-
-  if (process.env.JWT_SECRET === undefined) {
-    throw new Error('JWT_SECRET is not defined')
-  }
-  const auth = await signToken(email, process.env.JWT_SECRET)
-
-  if (process.env.INTERNAL_SVC_ENDPOINT === undefined) {
-    throw new Error('REST_BACKEND_ENDPOINT is not defined')
-  }
-  return axios.post(
-    `${process.env.INTERNAL_SVC_ENDPOINT}svc/email-attachment/create-article`,
-    data,
-    {
-      headers: {
-        Authorization: `${auth as string}`,
-        'Content-Type': 'application/json',
-      },
-    }
-  )
 }

packages/inbound-email-handler/src/index.ts

@@ -2,92 +2,29 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 /* eslint-disable @typescript-eslint/no-unsafe-argument */
 /* eslint-disable @typescript-eslint/no-unused-vars */
-
-import { PubSub } from '@google-cloud/pubsub'
-import { handleNewsletter } from '@omnivore/content-handler'
-import { generateUniqueUrl } from '@omnivore/content-handler/build/src/content-handler'
 import * as Sentry from '@sentry/serverless'
-import axios from 'axios'
-import * as jwt from 'jsonwebtoken'
 import parseHeaders from 'parse-headers'
 import * as multipart from 'parse-multipart-data'
 import rfc2047 from 'rfc2047'
-import { Converter } from 'showdown'
-import { promisify } from 'util'
 import { Attachment, handleAttachments, isAttachment } from './attachment'
+import { EmailJobType, queueEmailJob } from './job'
 import {
   handleGoogleConfirmationEmail,
   isGoogleConfirmationEmail,
   isSubscriptionConfirmationEmail,
-  parseAuthor,
   parseUnsubscribe,
 } from './newsletter'
 
-interface SaveReceivedEmailResponse {
-  id: string
-}
-
 interface Envelope {
   to: string[]
   from: string
 }
 
-const signToken = promisify(jwt.sign)
-
 Sentry.GCPFunction.init({
   dsn: process.env.SENTRY_DSN,
   tracesSampleRate: 0,
 })
 
-const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
-const NON_NEWSLETTER_EMAIL_TOPIC = 'nonNewsletterEmailReceived'
-const pubsub = new PubSub()
-const converter = new Converter()
-
-export const plainTextToHtml = (text: string): string => {
-  return converter.makeHtml(text)
-}
-
-export const publishMessage = async (
-  topic: string,
-  message: any
-): Promise<string | undefined> => {
-  return pubsub
-    .topic(topic)
-    .publishMessage({ json: message })
-    .catch((err) => {
-      console.log('error publishing message:', err)
-      return undefined
-    })
-}
-
-const saveReceivedEmail = async (
-  email: string,
-  data: any
-): Promise<SaveReceivedEmailResponse> => {
-  if (process.env.JWT_SECRET === undefined) {
-    throw new Error('JWT_SECRET is not defined')
-  }
-  const auth = await signToken(email, process.env.JWT_SECRET)
-
-  if (process.env.INTERNAL_SVC_ENDPOINT === undefined) {
-    throw new Error('REST_BACKEND_ENDPOINT is not defined')
-  }
-
-  const response = await axios.post(
-    `${process.env.INTERNAL_SVC_ENDPOINT}svc/pubsub/emails/save`,
-    data,
-    {
-      headers: {
-        Authorization: `${auth as string}`,
-        'Content-Type': 'application/json',
-      },
-    }
-  )
-
-  return response.data as SaveReceivedEmailResponse
-}
-
 export const parsedTo = (parsed: Record<string, string>): string => {
   // envelope to contains the real recipient email address
   try {
@@ -120,6 +57,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       // original sender email address
       const from = parsed['from']
+      const replyTo = parsed['reply-to']
       const subject = parsed['subject']
       const html = parsed['html']
       const text = parsed['text']
@@ -134,96 +72,77 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         ? parseUnsubscribe(unSubHeader)
         : undefined
 
-      const { id: receivedEmailId } = await saveReceivedEmail(to, {
-        from,
-        to,
-        subject,
-        html,
-        text,
-      })
-
       try {
         // check if it is a subscription or google confirmation email
         const isGoogleConfirmation = isGoogleConfirmationEmail(from, subject)
         if (isGoogleConfirmation || isSubscriptionConfirmationEmail(subject)) {
           console.debug('handleConfirmation', from, subject)
           // we need to parse the confirmation code from the email
-          isGoogleConfirmation &&
-            (await handleGoogleConfirmationEmail(to, subject))
-          // queue non-newsletter emails
-          await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({
-            json: {
-              from,
-              to,
-              subject,
-              html,
-              text,
-              unsubMailTo: unsubscribe?.mailTo,
-              unsubHttpUrl: unsubscribe?.httpUrl,
-              forwardedFrom,
-              receivedEmailId,
-            },
+          if (isGoogleConfirmation) {
+            await handleGoogleConfirmationEmail(from, to, subject)
+          }
+
+          // forward emails
+          await queueEmailJob(EmailJobType.ForwardEmail, {
+            from,
+            to,
+            subject,
+            html,
+            text,
+            headers,
+            forwardedFrom,
+            replyTo,
           })
           return res.send('ok')
         }
         if (attachments.length > 0) {
           console.debug('handle attachments', from, to, subject)
           // save the attachments as articles
-          await handleAttachments(to, subject, attachments, receivedEmailId)
+          await handleAttachments(from, to, subject, attachments)
           return res.send('ok')
         }
 
-        // convert text to html if html is not available
-        const content = html || plainTextToHtml(text)
-
         // all other emails are considered newsletters
-        const newsletterMessage = await handleNewsletter({
+        // queue newsletter emails
+        await queueEmailJob(EmailJobType.SaveNewsletter, {
           from,
           to,
           subject,
-          html: content,
+          html,
+          text,
           headers,
+          unsubMailTo: unsubscribe?.mailTo,
+          unsubHttpUrl: unsubscribe?.httpUrl,
+          forwardedFrom,
+          replyTo,
         })
 
-        // queue newsletter emails
-        await pubsub.topic(NEWSLETTER_EMAIL_RECEIVED_TOPIC).publishMessage({
-          json: {
-            email: to,
-            content,
-            url: generateUniqueUrl(),
-            title: subject,
-            author: parseAuthor(from),
-            unsubMailTo: unsubscribe?.mailTo,
-            unsubHttpUrl: unsubscribe?.httpUrl,
-            receivedEmailId,
-            ...newsletterMessage,
-          },
-        })
         res.send('newsletter received')
       } catch (error) {
-        console.log(
+        console.error(
           'error handling emails, will forward.',
           from,
           to,
           subject,
           error
         )
-        // queue error emails
-        await pubsub.topic(NON_NEWSLETTER_EMAIL_TOPIC).publishMessage({
-          json: {
-            from,
-            to,
-            subject,
-            html,
-            text,
-            forwardedFrom,
-            receivedEmailId,
-          },
+
+        // fallback to forward the email
+        await queueEmailJob(EmailJobType.ForwardEmail, {
+          from,
+          to,
+          subject,
+          html,
+          text,
+          headers,
+          forwardedFrom,
+          replyTo,
         })
+
         res.send('ok')
       }
     } catch (e) {
-      console.log(e)
+      console.error(e)
       res.send(e)
     }
   }

packages/inbound-email-handler/src/job.ts

@@ -0,0 +1,72 @@
+import { BulkJobOptions, Queue } from 'bullmq'
+import { redisDataSource } from './redis_data_source'
+
+const QUEUE_NAME = 'omnivore-backend-queue'
+export enum EmailJobType {
+  ForwardEmail = 'forward-email',
+  SaveNewsletter = 'save-newsletter',
+  ConfirmationEmail = 'confirmation-email',
+  SaveAttachment = 'save-attachment',
+}
+
+interface EmailJobData {
+  from: string
+  to: string
+  subject: string
+  html?: string
+  text?: string
+  headers?: Record<string, string | string[]>
+  unsubMailTo?: string
+  unsubHttpUrl?: string
+  forwardedFrom?: string
+  replyTo?: string
+  uploadFile?: {
+    fileName: string
+    contentType: string
+    id: string
+  }
+  confirmationCode?: string
+}
+
+const queue = new Queue(QUEUE_NAME, {
+  connection: redisDataSource.queueRedisClient,
+})
+
+const getPriority = (jobType: EmailJobType): number => {
+  // we want to prioritized jobs by the expected time to complete
+  // lower number means higher priority
+  // priority 1: jobs that are expected to finish immediately
+  // priority 5: jobs that are expected to finish in less than 10 second
+  // priority 10: jobs that are expected to finish in less than 10 minutes
+  // priority 100: jobs that are expected to finish in less than 1 hour
+  switch (jobType) {
+    case EmailJobType.ForwardEmail:
+    case EmailJobType.ConfirmationEmail:
+      return 1
+    case EmailJobType.SaveAttachment:
+    case EmailJobType.SaveNewsletter:
+      return 5
+    default:
+      throw new Error(`unknown job type: ${jobType as string}`)
+  }
+}
+
+const getOpts = (jobType: EmailJobType): BulkJobOptions => {
+  return {
+    removeOnComplete: true,
+    removeOnFail: true,
+    attempts: 3,
+    priority: getPriority(jobType),
+    backoff: {
+      type: 'exponential',
+      delay: 2000,
+    },
+  }
+}
+
+export const queueEmailJob = async (
+  jobType: EmailJobType,
+  data: EmailJobData
+) => {
+  await queue.add(jobType, data, getOpts(jobType))
+}

packages/inbound-email-handler/src/newsletter.ts

@@ -1,12 +1,11 @@
 import addressparser from 'addressparser'
-import { publishMessage } from './index'
+import { EmailJobType, queueEmailJob } from './job'
 
 interface Unsubscribe {
   mailTo?: string
   httpUrl?: string
 }
 
-const GOOGLE_CONFIRMATION_CODE_RECEIVED_TOPIC = 'emailConfirmationCodeReceived'
 const GOOGLE_CONFIRMATION_EMAIL_SENDER_ADDRESS = 'forwarding-noreply@google.com'
 // check unicode parentheses too
 const GOOGLE_CONFIRMATION_CODE_PATTERN = /\d+/u
@@ -41,24 +40,25 @@ export const parseAuthor = (address: string): string => {
 }
 
 export const handleGoogleConfirmationEmail = async (
-  email: string,
+  from: string,
+  to: string,
   subject: string
 ) => {
-  console.log('confirmation email', email, subject)
+  console.log('confirmation email', from, to, subject)
 
   const confirmationCode = getConfirmationCode(subject)
-  if (!email || !confirmationCode) {
+  if (!to || !confirmationCode) {
     console.log(
       'confirmation email error, user email:',
-      email,
+      to,
       'confirmationCode',
       confirmationCode
     )
     throw new Error('invalid confirmation email')
   }
 
-  const message = { emailAddress: email, confirmationCode: confirmationCode }
-  return publishMessage(GOOGLE_CONFIRMATION_CODE_RECEIVED_TOPIC, message)
+  const message = { from, to, confirmationCode, subject }
+  return queueEmailJob(EmailJobType.ConfirmationEmail, message)
 }
 
 export const getConfirmationCode = (subject: string): string | undefined => {

packages/inbound-email-handler/src/redis_data_source.ts

@@ -0,0 +1,99 @@
+import Redis, { RedisOptions } from 'ioredis'
+import 'dotenv/config'
+
+type RedisClientType = 'cache' | 'mq'
+type RedisDataSourceOption = {
+  url?: string
+  cert?: string
+}
+export type RedisDataSourceOptions = {
+  [key in RedisClientType]: RedisDataSourceOption
+}
+
+export class RedisDataSource {
+  options: RedisDataSourceOptions
+
+  cacheClient: Redis
+  queueRedisClient: Redis
+
+  constructor(options: RedisDataSourceOptions) {
+    this.options = options
+
+    const cacheClient = createIORedisClient('cache', this.options)
+    if (!cacheClient) throw 'Error initializing cache redis client'
+
+    this.cacheClient = cacheClient
+    this.queueRedisClient =
+      createIORedisClient('mq', this.options) || this.cacheClient // if mq is not defined, use cache
+  }
+
+  async shutdown(): Promise<void> {
+    try {
+      await this.queueRedisClient?.quit()
+      await this.cacheClient?.quit()
+    } catch (err) {
+      console.error('error while shutting down redis', err)
+    }
+  }
+}
+
+const createIORedisClient = (
+  name: RedisClientType,
+  options: RedisDataSourceOptions
+): Redis | undefined => {
+  const option = options[name]
+  const redisURL = option.url
+  if (!redisURL) {
+    console.log(`no redisURL supplied: ${name}`)
+    return undefined
+  }
+
+  const redisCert = option.cert
+  const tls =
+    redisURL.startsWith('rediss://') && redisCert
+      ? {
+          ca: redisCert,
+          rejectUnauthorized: false,
+        }
+      : undefined
+
+  const redisOptions: RedisOptions = {
+    tls,
+    name,
+    connectTimeout: 10000,
+    maxRetriesPerRequest: null,
+    offlineQueue: false,
+  }
+  return new Redis(redisURL, redisOptions)
+}
+
+export const redisDataSource = new RedisDataSource({
+  cache: {
+    url: process.env.REDIS_URL,
+    cert: process.env.REDIS_CERT,
+  },
+  mq: {
+    url: process.env.MQ_REDIS_URL,
+    cert: process.env.MQ_REDIS_CERT,
+  },
+})
+
+const gracefulShutdown = async (signal: string) => {
+  console.log(`Received ${signal}, shutting down gracefully...`)
+
+  await redisDataSource.shutdown()
+  console.log('redis shutdown successfully')
+
+  process.exit(0)
+}
+
+process.on('SIGINT', () => {
+  ;(async () => {
+    await gracefulShutdown('SIGINT')
+  })()
+})
+process.on('SIGTERM', () => {
+  ;(async () => {
+    await gracefulShutdown('SIGTERM')
+  })()
+})

packages/inbound-email-handler/test/global-teardown.ts

@@ -0,0 +1,5 @@
+import { redisDataSource } from '../src/redis_data_source'
+
+export const mochaGlobalTeardown = async () => {
+  await redisDataSource.shutdown()
+}

packages/inbound-email-handler/test/newsletter.test.ts

@@ -2,7 +2,7 @@ import { expect } from 'chai'
 import 'mocha'
 import parseHeaders from 'parse-headers'
 import rfc2047 from 'rfc2047'
-import { parsedTo, plainTextToHtml } from '../src'
+import { parsedTo } from '../src'
 import {
   getConfirmationCode,
   isGoogleConfirmationEmail,
@@ -138,29 +138,3 @@ describe('decode and parse headers', () => {
     })
   })
 })
-
-describe('plainTextToHtml', () => {
-  it('converts text to html', () => {
-    const text =
-      'DEVOPS WEEKLY\r\n' +
-      'ISSUE #665 - 24th September 2023\r\n' +
-      '\r\n' +
-      'A few posts on CI tooling this week, along with a good introduction to developer portals/platforms and other topics.\r\n' +
-      '\r\n' +
-      'StackHawk sponsors Devops Weekly\r\n' +
-      '============================\r\n' +
-      '\r\n' +
-      'Experience automated security testing without the hassle of connecting your own app or configuring an environment! Follow the Tutorial to try out StackHawk and explore a world where security becomes an accelerator, not a blocker\r\n' +
-      '\r\n' +
-      'https://sthwk.com/tutorial\r\n' +
-      '\r\n'
-    expect(plainTextToHtml(text)).to.eql(
-      `<p>DEVOPS WEEKLY
-ISSUE #665 - 24th September 2023</p>
-<p>A few posts on CI tooling this week, along with a good introduction to developer portals/platforms and other topics.</p>
-<h1 id="stackhawksponsorsdevopsweekly">StackHawk sponsors Devops Weekly</h1>
-<p>Experience automated security testing without the hassle of connecting your own app or configuring an environment! Follow the Tutorial to try out StackHawk and explore a world where security becomes an accelerator, not a blocker</p>
-<p>https://sthwk.com/tutorial</p>`
-    )
-  })
-})

packages/inbound-email-handler/tsconfig.json

@@ -5,5 +5,5 @@
     "rootDir": ".",
     "lib": ["dom"]
   },
-  "include": ["src"]
+  "include": ["src", "test"]
 }

packages/pdf-handler/package.json

@@ -34,6 +34,7 @@
     "axios": "^0.27.2",
     "bullmq": "^5.1.4",
     "concurrently": "^7.0.0",
+    "dotenv": "^8.2.0",
     "ioredis": "^5.3.2",
     "pdfjs-dist": "^2.9.359"
   },

packages/pdf-handler/src/index.ts

@@ -1,7 +1,7 @@
 import { GetSignedUrlConfig, Storage } from '@google-cloud/storage'
 import * as Sentry from '@sentry/serverless'
 import { parsePdf } from './pdf'
-import { queueUpdatePageJob } from './job'
+import { queueUpdatePageJob, State } from './job'
 
 Sentry.GCPFunction.init({
   dsn: process.env.SENTRY_DSN,
@@ -50,10 +50,11 @@ const getDocumentUrl = async (
 
 export const updatePageContent = async (
   fileId: string,
-  content: string,
+  content?: string,
   title?: string,
   author?: string,
-  description?: string
+  description?: string,
+  state?: State
 ): Promise<string | undefined> => {
   const job = await queueUpdatePageJob({
     fileId,
@@ -61,6 +62,7 @@ export const updatePageContent = async (
     title,
     author,
     description,
+    state,
   })
   return job.id
 }
@@ -86,45 +88,68 @@ export const pdfHandler = Sentry.GCPFunction.wrapHttpFunction(
     if ('message' in req.body && 'data' in req.body.message) {
       const pubSubMessage = req.body.message.data as string
       const data = getStorageEventData(pubSubMessage)
-      if (data) {
-        try {
-          if (shouldHandle(data)) {
-            console.log('handling pdf data', data)
-
-            const url = await getDocumentUrl(data)
-            console.log('PDF url: ', url)
-            if (!url) {
-              console.log('Could not fetch PDF', data.bucket, data.name)
-              return res.status(404).send('Could not fetch PDF')
-            }
-
-            const parsed = await parsePdf(url)
-            const result = await updatePageContent(
-              data.name,
-              parsed.content,
-              parsed.title,
-              parsed.author,
-              parsed.description
-            )
-            console.log(
-              'publish result',
-              result,
-              'title',
-              parsed.title,
-              'author',
-              parsed.author
-            )
-          } else {
-            console.log('not handling pdf data', data)
-          }
-        } catch (err) {
-          console.log('error handling event', { err, data })
-          return res.status(500).send('Error handling event')
-        }
+      if (!data) {
+        console.log('no data found in pubsub message')
+        return res.send('ok')
+      }
+
+      if (!shouldHandle(data)) {
+        console.log('not handling pdf data', data)
+        return res.send('ok')
+      }
+
+      console.log('handling pdf data', data)
+
+      let content,
+        title,
+        author,
+        description,
+        state: State = 'SUCCEEDED' // Default to succeeded even if we fail to parse
+
+      try {
+        const url = await getDocumentUrl(data)
+        console.log('PDF url: ', url)
+        if (!url) {
+          console.log('Could not fetch PDF', data.bucket, data.name)
+          // If we can't fetch the PDF, mark it as failed
+          state = 'FAILED'
+
+          return res.status(404).send('Could not fetch PDF')
+        }
+
+        // Parse the PDF to update the content and metadata
+        const parsed = await parsePdf(url)
+        content = parsed.content
+        title = parsed.title
+        author = parsed.author
+        description = parsed.description
+      } catch (err) {
+        console.log('error parsing pdf', { err, data })
+
+        return res.status(500).send('Error parsing pdf')
+      } finally {
+        // Always update the state, even if we fail to parse
+        const result = await updatePageContent(
+          data.name,
+          content,
+          title,
+          author,
+          description,
+          state
+        )
+        console.log(
+          'publish result',
+          result,
+          'title',
+          title,
+          'author',
+          author,
+          'state',
+          state
+        )
       }
-    } else {
-      console.log('no pubsub message')
     }
+
     res.send('ok')
   }
 )

packages/pdf-handler/src/job.ts

@@ -8,12 +8,15 @@ const queue = new Queue(QUEUE_NAME, {
   connection: redisDataSource.queueRedisClient,
 })
 
+export type State = 'SUCCEEDED' | 'FAILED'
+
 type UpdatePageJobData = {
   fileId: string
-  content: string
+  content?: string
   title?: string
   author?: string
   description?: string
+  state?: State
 }
 
 export const queueUpdatePageJob = async (data: UpdatePageJobData) => {

packages/pdf-handler/src/redis_data_source.ts

@@ -1,4 +1,5 @@
 import Redis, { RedisOptions } from 'ioredis'
+import 'dotenv/config'
 
 export type RedisDataSourceOptions = {
   REDIS_URL?: string

packages/text-to-speech/package.json

@@ -18,11 +18,11 @@
     "start": "functions-framework --target=textToSpeechHandler",
     "start_streaming": "functions-framework --target=textToSpeechStreamingHandler",
     "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"",
+    "dev_streaming": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start_streaming\"",
     "gcloud-deploy": "gcloud functions deploy text-to-speech --gen2 --entry-point=textToSpeechHandler --trigger-http --allow-unauthenticated --region=us-west2 --runtime nodejs14",
     "deploy": "yarn build && yarn gcloud-deploy"
   },
   "devDependencies": {
-    "@types/fluent-ffmpeg": "^2.1.20",
     "@types/html-to-text": "^8.1.1",
     "@types/natural": "^5.1.1",
     "@types/node": "^14.11.2",
@@ -32,19 +32,18 @@
     "mocha": "^10.0.0"
   },
   "dependencies": {
-    "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "@google-cloud/functions-framework": "3.1.2",
     "@google-cloud/storage": "^7.0.1",
     "@sentry/serverless": "^7.77.0",
     "axios": "^0.27.2",
     "dotenv": "^16.0.1",
-    "fluent-ffmpeg": "^2.1.2",
     "html-to-text": "^8.2.1",
     "ioredis": "^5.3.2",
     "jsonwebtoken": "^8.5.1",
     "linkedom": "^0.14.12",
     "microsoft-cognitiveservices-speech-sdk": "1.30",
     "natural": "^6.2.0",
+    "nodemon": "^2.0.15",
     "underscore": "^1.13.4"
   },
   "volta": {

packages/text-to-speech/src/index.ts

@@ -232,8 +232,10 @@ export const textToSpeechStreamingHandler = Sentry.GCPFunction.wrapHttpFunction(
 
     let claim: Claim
     try {
-      jwt.verify(token, process.env.JWT_SECRET)
-      claim = jwt.decode(token) as Claim
+      // ignore expiration for now and verify function will also decode the token
+      claim = jwt.verify(token, process.env.JWT_SECRET, {
+        ignoreExpiration: true,
+      }) as Claim
     } catch (e) {
       console.error('Authentication error:', e)
       return res.status(401).send({ errorCode: 'UNAUTHENTICATED' })