Merge branch 'main' into OMN-190

.github/workflows/run-tests.yaml

@@ -69,7 +69,6 @@ jobs:
           yarn build
           yarn lint
           yarn test
-
         env:
           PG_HOST: localhost
           PG_PORT: ${{ job.services.postgres.ports[5432] }}
@@ -78,3 +77,12 @@ jobs:
           PG_DB: omnivore_test
           PG_POOL_MAX: 10
           ELASTIC_URL: http://localhost:${{ job.services.elastic.ports[9200] }}/
+  build-docker-images:
+    name: Build docker images
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Build the API docker image
+        run: 'docker build --file packages/api/Dockerfile .'

apple/OmnivoreKit/Sources/App/Views/ApplyLabelsView.swift

@@ -0,0 +1,87 @@
+import Combine
+import Models
+import Services
+import SwiftUI
+import Views
+
+final class ApplyLabelsViewModel: ObservableObject {
+  private var hasLoadedInitialLabels = false
+  @Published var isLoading = true
+  @Published var selectedLabels = Set<FeedItemLabel>()
+  @Published var labels = [FeedItemLabel]()
+
+  var subscriptions = Set<AnyCancellable>()
+
+  func load(item: FeedItem, dataService: DataService) {
+    guard !hasLoadedInitialLabels else { return }
+
+    dataService.labelsPublisher().sink(
+      receiveCompletion: { _ in },
+      receiveValue: { [weak self] result in
+        self?.isLoading = false
+        self?.labels = result
+        self?.hasLoadedInitialLabels = true
+        self?.selectedLabels = Set(item.labels)
+      }
+    )
+    .store(in: &subscriptions)
+  }
+
+  func saveChanges(itemID: String, dataService: DataService, onComplete: @escaping ([FeedItemLabel]) -> Void) {
+    dataService.updateArticleLabelsPublisher(itemID: itemID, labelIDs: selectedLabels.map(\.id)).sink(
+      receiveCompletion: { _ in },
+      receiveValue: { onComplete($0) }
+    )
+    .store(in: &subscriptions)
+  }
+}
+
+struct ApplyLabelsView: View {
+  let item: FeedItem
+  let commitLabelChanges: ([FeedItemLabel]) -> Void
+
+  @EnvironmentObject var dataService: DataService
+  @Environment(\.presentationMode) private var presentationMode
+  @StateObject var viewModel = ApplyLabelsViewModel()
+
+  var body: some View {
+    NavigationView {
+      if viewModel.isLoading {
+        EmptyView()
+      } else {
+        List(viewModel.labels, id: \.self, selection: $viewModel.selectedLabels) { label in
+          if let textChip = TextChip(feedItemLabel: label) {
+            textChip
+          } else {
+            Text(label.name)
+          }
+        }
+        .environment(\.editMode, .constant(EditMode.active))
+        .navigationTitle("Apply Labels")
+        .navigationBarTitleDisplayMode(.inline)
+        .toolbar {
+          ToolbarItem(placement: .navigationBarLeading) {
+            Button(
+              action: { presentationMode.wrappedValue.dismiss() },
+              label: { Text("Cancel") }
+            )
+          }
+          ToolbarItem(placement: .navigationBarTrailing) {
+            Button(
+              action: {
+                viewModel.saveChanges(itemID: item.id, dataService: dataService) { labels in
+                  commitLabelChanges(labels)
+                  presentationMode.wrappedValue.dismiss()
+                }
+              },
+              label: { Text("Save") }
+            )
+          }
+        }
+      }
+    }
+    .onAppear {
+      viewModel.load(item: item, dataService: dataService)
+    }
+  }
+}

apple/OmnivoreKit/Sources/App/Views/Home/Components/FeedCardNavigationLink.swift

@@ -36,7 +36,6 @@ struct GridCardNavigationLink: View {
   @EnvironmentObject var dataService: DataService
 
   @State private var scale = 1.0
-  @State private var isActive = false
 
   let item: FeedItem
   let searchQuery: String
@@ -51,7 +50,8 @@ struct GridCardNavigationLink: View {
     ZStack {
       NavigationLink(
         destination: LinkItemDetailView(viewModel: LinkItemDetailViewModel(item: item, homeFeedViewModel: viewModel)),
-        isActive: $isActive
+        tag: item,
+        selection: $selectedLinkItem
       ) {
         EmptyView()
       }
@@ -60,7 +60,7 @@ struct GridCardNavigationLink: View {
           scale = 0.95
           DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(150)) {
             scale = 1.0
-            isActive = true
+            selectedLinkItem = item
           }
         }
       })
@@ -68,7 +68,7 @@ struct GridCardNavigationLink: View {
           viewModel.itemAppeared(item: item, searchQuery: searchQuery, dataService: dataService)
         }
     }
-    .aspectRatio(2.1, contentMode: .fill)
+    .aspectRatio(1.8, contentMode: .fill)
     .scaleEffect(scale)
   }
 }

apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewIOS.swift

@@ -100,6 +100,9 @@ import Views
           viewModel.loadItems(dataService: dataService, searchQuery: searchQuery, isRefresh: true)
         }
       }
+      .onChange(of: selectedLinkItem) { _ in
+        viewModel.commitProgressUpdates()
+      }
     }
   }
 
@@ -164,6 +167,11 @@ import Views
             }
           }
         }
+        .sheet(item: $viewModel.itemUnderLabelEdit) { item in
+          ApplyLabelsView(item: item) { labels in
+            viewModel.updateLabels(itemID: item.id, labels: labels)
+          }
+        }
       }
     }
   }
@@ -320,13 +328,15 @@ import Views
       case .delete:
         itemToRemove = item
         confirmationShown = true
+      case .editLabels:
+        viewModel.itemUnderLabelEdit = item
       }
     }
 
     var body: some View {
       ScrollView {
         LazyVGrid(columns: [GridItem(.adaptive(minimum: 325), spacing: 24)], spacing: 24) {
-          ForEach(viewModel.items, id: \.renderID) { item in
+          ForEach(viewModel.items) { item in
             let link = GridCardNavigationLink(
               item: item,
               searchQuery: searchQuery,

apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift

@@ -8,9 +8,13 @@ import Views
 final class HomeFeedViewModel: ObservableObject {
   var currentDetailViewModel: LinkItemDetailViewModel?
 
+  /// Track progress updates to be committed when user navigates back to grid view
+  var uncommittedReadingProgressUpdates = [String: Double]()
+
   @Published var items = [FeedItem]()
   @Published var isLoading = false
   @Published var showPushNotificationPrimer = false
+  @Published var itemUnderLabelEdit: FeedItem?
   var cursor: String?
   var sendProgressUpdates = false
 
@@ -76,6 +80,9 @@ final class HomeFeedViewModel: ObservableObject {
         if thisSearchIdx > 0, thisSearchIdx <= self?.receivedIdx ?? 0 {
           return
         }
+
+        dataService.prefetchPages(items: result.items)
+
         self?.items = isRefresh ? result.items : (self?.items ?? []) + result.items
         self?.isLoading = false
         self?.receivedIdx = thisSearchIdx
@@ -160,10 +167,27 @@ final class HomeFeedViewModel: ObservableObject {
     .store(in: &subscriptions)
   }
 
-  func updateProgress(itemID: String, progress: Double) {
+  /// Update `FeedItem`s with the cached reading progress values so it can animate when the
+  /// user navigates back to the grid view (and also avoid mutations of the grid items
+  /// that can cause the `NavigationView` to pop.
+  func commitProgressUpdates() {
+    for (key, value) in uncommittedReadingProgressUpdates {
+      updateProgress(itemID: key, progress: value)
+    }
+    uncommittedReadingProgressUpdates = [:]
+  }
+
+  private func updateProgress(itemID: String, progress: Double) {
     guard sendProgressUpdates, let item = items.first(where: { $0.id == itemID }) else { return }
     if let index = items.firstIndex(of: item) {
       items[index].readingProgress = progress
     }
   }
+
+  func updateLabels(itemID: String, labels: [FeedItemLabel]) {
+    guard let item = items.first(where: { $0.id == itemID }) else { return }
+    if let index = items.firstIndex(of: item) {
+      items[index].labels = labels
+    }
+  }
 }

apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift

@@ -92,7 +92,7 @@ final class LinkItemDetailViewModel: ObservableObject {
       case let .shareHighlight(highlightID):
         print("show share modal for highlight with id: \(highlightID)")
       case let .updateReadingProgess(progress: progress):
-        self?.homeFeedViewModel.updateProgress(itemID: self?.item.id ?? "", progress: Double(progress))
+        self?.homeFeedViewModel.uncommittedReadingProgressUpdates[self?.item.id ?? ""] = Double(progress)
       }
     }
     .store(in: &newWebAppWrapperViewModel.subscriptions)

apple/OmnivoreKit/Sources/App/Views/Profile/LabelsView.swift

@@ -0,0 +1,195 @@
+import Combine
+import Models
+import Services
+import SwiftUI
+import Views
+
+final class LabelsViewModel: ObservableObject {
+  private var hasLoadedInitialLabels = false
+  @Published var isLoading = false
+  @Published var labels = [FeedItemLabel]()
+  @Published var showCreateEmailModal = false
+
+  var subscriptions = Set<AnyCancellable>()
+
+  func loadLabels(dataService: DataService) {
+    guard !hasLoadedInitialLabels else { return }
+    isLoading = true
+
+    dataService.labelsPublisher().sink(
+      receiveCompletion: { _ in },
+      receiveValue: { [weak self] result in
+        self?.isLoading = false
+        self?.labels = result
+        self?.hasLoadedInitialLabels = true
+      }
+    )
+    .store(in: &subscriptions)
+  }
+
+  func createLabel(dataService: DataService, name: String, color: Color, description: String?) {
+    isLoading = true
+
+    dataService.createLabelPublisher(
+      name: name,
+      color: color.hex ?? "",
+      description: description
+    ).sink(
+      receiveCompletion: { [weak self] _ in
+        self?.isLoading = false
+      },
+      receiveValue: { [weak self] result in
+        self?.isLoading = false
+        self?.labels.insert(result, at: 0)
+        self?.showCreateEmailModal = false
+      }
+    )
+    .store(in: &subscriptions)
+  }
+
+  func deleteLabel(dataService: DataService, labelID: String) {
+    isLoading = true
+
+    dataService.removeLabelPublisher(labelID: labelID).sink(
+      receiveCompletion: { [weak self] _ in
+        self?.isLoading = false
+      },
+      receiveValue: { [weak self] _ in
+        self?.isLoading = false
+        self?.labels.removeAll { $0.id == labelID }
+      }
+    )
+    .store(in: &subscriptions)
+  }
+}
+
+struct LabelsView: View {
+  @EnvironmentObject var dataService: DataService
+  @StateObject var viewModel = LabelsViewModel()
+  @State private var showDeleteConfirmation = false
+  @State private var labelToRemoveID: String?
+
+  let footerText = "Use labels to create curated collections of links."
+
+  var body: some View {
+    Group {
+      #if os(iOS)
+        if #available(iOS 15.0, *) {
+          Form {
+            innerBody
+              .alert("Are you sure you want to delete this label?", isPresented: $showDeleteConfirmation) {
+                Button("Remove Link", role: .destructive) {
+                  if let labelID = labelToRemoveID {
+                    withAnimation {
+                      viewModel.deleteLabel(dataService: dataService, labelID: labelID)
+                    }
+                  }
+                  self.labelToRemoveID = nil
+                }
+                Button("Cancel", role: .cancel) { self.labelToRemoveID = nil }
+              }
+          }
+        } else {
+          Form { innerBody }
+        }
+
+      #elseif os(macOS)
+        List {
+          innerBody
+        }
+        .listStyle(InsetListStyle())
+      #endif
+    }
+    .onAppear { viewModel.loadLabels(dataService: dataService) }
+  }
+
+  private var innerBody: some View {
+    Group {
+      Section(footer: Text(footerText)) {
+        Button(
+          action: { viewModel.showCreateEmailModal = true },
+          label: {
+            HStack {
+              Image(systemName: "plus.circle.fill").foregroundColor(.green)
+              Text("Create a new Label")
+              Spacer()
+            }
+          }
+        )
+        .disabled(viewModel.isLoading)
+      }
+
+      if !viewModel.labels.isEmpty {
+        Section(header: Text("Labels")) {
+          ForEach(viewModel.labels, id: \.id) { label in
+            HStack {
+              Text(label.name)
+              Spacer()
+              Button(
+                action: {
+                  labelToRemoveID = label.id
+                  showDeleteConfirmation = true
+                },
+                label: { Image(systemName: "trash") }
+              )
+            }
+          }
+        }
+      }
+    }
+    .navigationTitle("Labels")
+    .sheet(isPresented: $viewModel.showCreateEmailModal) {
+      CreateLabelView(viewModel: viewModel)
+    }
+  }
+}
+
+struct CreateLabelView: View {
+  @EnvironmentObject var dataService: DataService
+  @ObservedObject var viewModel: LabelsViewModel
+
+  @State private var newLabelName = ""
+  @State private var newLabelColor = Color.clear
+
+  var body: some View {
+    NavigationView {
+      VStack(spacing: 16) {
+        TextField("Label Name", text: $newLabelName)
+          .keyboardType(.alphabet)
+          .textFieldStyle(StandardTextFieldStyle())
+        ColorPicker(
+          newLabelColor == .clear ? "Select Color" : newLabelColor.description,
+          selection: $newLabelColor
+        )
+        Button(
+          action: {
+            viewModel.createLabel(
+              dataService: dataService,
+              name: newLabelName,
+              color: newLabelColor,
+              description: nil
+            )
+          },
+          label: { Text("Create") }
+        )
+        .buttonStyle(SolidCapsuleButtonStyle(color: .appDeepBackground, width: 300))
+        .disabled(viewModel.isLoading || newLabelName.isEmpty || newLabelColor == .clear)
+        Spacer()
+      }
+      .padding()
+      .toolbar {
+        ToolbarItem(placement: .automatic) {
+          Button(
+            action: { viewModel.showCreateEmailModal = false },
+            label: {
+              Image(systemName: "xmark")
+                .foregroundColor(.appGrayTextContrast)
+            }
+          )
+        }
+      }
+      .navigationTitle("Create New Label")
+      .navigationBarTitleDisplayMode(.inline)
+    }
+  }
+}

apple/OmnivoreKit/Sources/App/Views/Profile/ProfileView.swift

@@ -63,6 +63,12 @@ struct ProfileView: View {
       }
 
       Section {
+        if FeatureFlag.enableLabels {
+          NavigationLink(destination: LabelsView()) {
+            Text("Labels")
+          }
+        }
+
         NavigationLink(destination: NewsletterEmailsView()) {
           Text("Emails")
         }

apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderContainer.swift

@@ -36,7 +36,7 @@ struct WebReaderContainerView: View {
       let messageBody = message.body as? [String: Double]
 
       if let messageBody = messageBody, let progress = messageBody["progress"] {
-        homeFeedViewModel.updateProgress(itemID: item.id, progress: Double(progress))
+        homeFeedViewModel.uncommittedReadingProgressUpdates[item.id] = Double(progress)
       }
     }
 
@@ -56,7 +56,7 @@ struct WebReaderContainerView: View {
     if message.name == WebViewAction.readingProgressUpdate.rawValue {
       guard let messageBody = message.body as? [String: Double] else { return }
       guard let progress = messageBody["progress"] else { return }
-      homeFeedViewModel.updateProgress(itemID: item.id, progress: Double(progress))
+      homeFeedViewModel.uncommittedReadingProgressUpdates[item.id] = Double(progress)
     }
   }
 
@@ -204,7 +204,7 @@ struct WebReaderContainerView: View {
               Color.systemBackground
                 .transition(.opacity)
                 .onAppear {
-                  DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(250)) {
+                  DispatchQueue.main.asyncAfter(deadline: .now() + .milliseconds(100)) {
                     withAnimation(.linear(duration: 0.2)) {
                       showOverlay = false
                     }

apple/OmnivoreKit/Sources/App/Views/WebReader/WebReaderViewModel.swift

@@ -10,24 +10,28 @@ struct SafariWebLink: Identifiable {
 }
 
 func encodeHighlightResult(_ highlight: Highlight) -> [String: Any]? {
-  let data = try? JSONEncoder().encode(highlight)
-  if let data = data, let dictionary = try? JSONSerialization.jsonObject(with: data, options: .allowFragments) as? [String: Any] {
-    return dictionary
-  }
-  return nil
+  guard let data = try? JSONEncoder().encode(highlight) else { return nil }
+  return try? JSONSerialization.jsonObject(with: data, options: .allowFragments) as? [String: Any]
 }
 
 final class WebReaderViewModel: ObservableObject {
   @Published var isLoading = false
   @Published var articleContent: ArticleContent?
 
+  var slug: String?
   var subscriptions = Set<AnyCancellable>()
 
   func loadContent(dataService: DataService, slug: String) {
+    self.slug = slug
     isLoading = true
 
     guard let viewer = dataService.currentViewer else { return }
 
+    if let content = dataService.pageFromCache(slug: slug) {
+      articleContent = content
+      // continue to load from the web if possible
+    }
+
     dataService.articleContentPublisher(username: viewer.username, slug: slug).sink(
       receiveCompletion: { [weak self] completion in
         guard case .failure = completion else { return }
@@ -35,6 +39,7 @@ final class WebReaderViewModel: ObservableObject {
       },
       receiveValue: { [weak self] articleContent in
         self?.articleContent = articleContent
+        dataService.pageCache.setObject(CachedPageContent(slug, articleContent), forKey: NSString(string: slug))
       }
     )
     .store(in: &subscriptions)
@@ -167,12 +172,16 @@ final class WebReaderViewModel: ObservableObject {
 
     switch actionID {
     case "deleteHighlight":
+      dataService.invalidateCachedPage(slug: slug)
       deleteHighlight(messageBody: messageBody, replyHandler: replyHandler, dataService: dataService)
     case "createHighlight":
+      dataService.invalidateCachedPage(slug: slug)
       createHighlight(messageBody: messageBody, replyHandler: replyHandler, dataService: dataService)
     case "mergeHighlight":
+      dataService.invalidateCachedPage(slug: slug)
       mergeHighlight(messageBody: messageBody, replyHandler: replyHandler, dataService: dataService)
     case "updateHighlight":
+      dataService.invalidateCachedPage(slug: slug)
       updateHighlight(messageBody: messageBody, replyHandler: replyHandler, dataService: dataService)
     case "articleReadingProgress":
       updateReadingProgress(messageBody: messageBody, replyHandler: replyHandler, dataService: dataService)

apple/OmnivoreKit/Sources/Models/ArticleContent.swift

@@ -1,5 +1,15 @@
 import Foundation
 
+public class CachedPageContent: NSObject {
+  public let slug: String
+  public let value: ArticleContent
+
+  public init(_ slug: String, _ content: ArticleContent) {
+    self.slug = slug
+    self.value = content
+  }
+}
+
 public struct ArticleContent {
   public let htmlContent: String
   public let highlights: [Highlight]

apple/OmnivoreKit/Sources/Models/FeedItem.swift

@@ -12,7 +12,6 @@ public struct HomeFeedData {
 
 public struct FeedItem: Identifiable, Hashable, Decodable {
   public let id: String
-  public let renderID = UUID()
   public let title: String
   public let createdAt: Date
   public let savedAt: Date
@@ -29,6 +28,7 @@ public struct FeedItem: Identifiable, Hashable, Decodable {
   public let slug: String
   public let isArchived: Bool
   public let contentReader: String?
+  public var labels: [FeedItemLabel]
 
   public init(
     id: String,
@@ -47,7 +47,8 @@ public struct FeedItem: Identifiable, Hashable, Decodable {
     publishDate: Date?,
     slug: String,
     isArchived: Bool,
-    contentReader: String?
+    contentReader: String?,
+    labels: [FeedItemLabel]
   ) {
     self.id = id
     self.title = title
@@ -66,10 +67,12 @@ public struct FeedItem: Identifiable, Hashable, Decodable {
     self.slug = slug
     self.isArchived = isArchived
     self.contentReader = contentReader
+    self.labels = labels
   }
 
   enum CodingKeys: String, CodingKey {
-    case id, title, createdAt, savedAt, image, isArchived, readingProgressPercent, readingProgressAnchorIndex, slug, contentReader, url
+    // swiftlint:disable:next line_length
+    case id, title, createdAt, savedAt, image, isArchived, readingProgressPercent, readingProgressAnchorIndex, slug, contentReader, url, labels
   }
 
   public init(from decoder: Decoder) throws {
@@ -86,6 +89,7 @@ public struct FeedItem: Identifiable, Hashable, Decodable {
     contentReader = try container.decode(String.self, forKey: .contentReader)
     pageURLString = try container.decode(String.self, forKey: .url)
     isArchived = try container.decode(Bool.self, forKey: .isArchived)
+    labels = try container.decode([FeedItemLabel].self, forKey: .labels)
 
     self.onDeviceImageURLString = nil
     self.documentDirectoryPath = nil

apple/OmnivoreKit/Sources/Models/FeedItemLabel.swift

@@ -0,0 +1,23 @@
+import Foundation
+
+public struct FeedItemLabel: Decodable, Hashable {
+  public let id: String
+  public let name: String
+  public let color: String
+  public let createdAt: Date?
+  public let description: String?
+
+  public init(
+    id: String,
+    name: String,
+    color: String,
+    createdAt: Date?,
+    description: String?
+  ) {
+    self.id = id
+    self.name = name
+    self.color = color
+    self.createdAt = createdAt
+    self.description = description
+  }
+}

apple/OmnivoreKit/Sources/Services/DataService/DataService.swift

@@ -1,6 +1,16 @@
+import Combine
 import Foundation
 import Models
 
+public class CacheManager: NSObject, NSCacheDelegate {
+  public func cache(_: NSCache<AnyObject, AnyObject>, willEvictObject obj: Any) {
+    // This is just used for debugging
+    if let content = obj as? CachedPageContent {
+      print("evicting page from cache", content.slug)
+    }
+  }
+}
+
 public final class DataService: ObservableObject {
   public static var registerIntercomUser: ((String) -> Void)?
   public static var showIntercomMessenger: (() -> Void)?
@@ -9,12 +19,20 @@ public final class DataService: ObservableObject {
   public internal(set) var currentViewer: Viewer?
   let networker: Networker
 
+  public let pageCache = NSCache<NSString, CachedPageContent>()
+  let pageCacheQueue = DispatchQueue.global(qos: .background)
+
   let highlightsCache = NSCache<AnyObject, CachedPDFHighlights>()
   let highlightsCacheQueue = DispatchQueue(label: "app.omnivore.highlights.cache.queue", attributes: .concurrent)
 
+  let cacheManager: CacheManager
+  var subscriptions = Set<AnyCancellable>()
+
   public init(appEnvironment: AppEnvironment, networker: Networker) {
     self.appEnvironment = appEnvironment
     self.networker = networker
+    self.cacheManager = CacheManager()
+    pageCache.delegate = cacheManager
   }
 
   public func clearHighlights() {
@@ -30,3 +48,37 @@ public final class DataService: ObservableObject {
     }
   }
 }
+
+public extension DataService {
+  func prefetchPages(items: [FeedItem]) {
+    print("prefetching pages")
+    guard let viewer = currentViewer else { return }
+
+    for item in items {
+      let slug = item.slug
+      articleContentPublisher(username: viewer.username, slug: slug).sink(
+        receiveCompletion: { _ in },
+        receiveValue: { [weak self] articleContent in
+          self?.pageCache.setObject(CachedPageContent(slug, articleContent), forKey: NSString(string: slug))
+        }
+      )
+      .store(in: &subscriptions)
+    }
+  }
+
+  func pageFromCache(slug: String) -> ArticleContent? {
+    if let content = pageCache.object(forKey: NSString(string: slug)) {
+      print("cache hit", slug)
+      return content.value
+    } else {
+      print("cache miss", slug)
+    }
+    return nil
+  }
+
+  func invalidateCachedPage(slug: String?) {
+    if let slug = slug {
+      pageCache.removeObject(forKey: NSString(string: slug))
+    }
+  }
+}

apple/OmnivoreKit/Sources/Services/DataService/GQLSchema.swift

@@ -2970,8 +2970,11 @@ extension Objects {
     let savedByViewer: [String: Bool]
     let shareInfo: [String: Objects.LinkShareInfo]
     let sharedComment: [String: String]
+    let siteIcon: [String: String]
+    let siteName: [String: String]
     let slug: [String: String]
     let title: [String: String]
+    let uploadFileId: [String: String]
     let url: [String: String]
 
     enum TypeName: String, Codable {
@@ -3088,6 +3091,14 @@ extension Objects.Article: Decodable {
         if let value = try container.decode(String?.self, forKey: codingKey) {
           map.set(key: field, hash: alias, value: value as Any)
         }
+      case "siteIcon":
+        if let value = try container.decode(String?.self, forKey: codingKey) {
+          map.set(key: field, hash: alias, value: value as Any)
+        }
+      case "siteName":
+        if let value = try container.decode(String?.self, forKey: codingKey) {
+          map.set(key: field, hash: alias, value: value as Any)
+        }
       case "slug":
         if let value = try container.decode(String?.self, forKey: codingKey) {
           map.set(key: field, hash: alias, value: value as Any)
@@ -3096,6 +3107,10 @@ extension Objects.Article: Decodable {
         if let value = try container.decode(String?.self, forKey: codingKey) {
           map.set(key: field, hash: alias, value: value as Any)
         }
+      case "uploadFileId":
+        if let value = try container.decode(String?.self, forKey: codingKey) {
+          map.set(key: field, hash: alias, value: value as Any)
+        }
       case "url":
         if let value = try container.decode(String?.self, forKey: codingKey) {
           map.set(key: field, hash: alias, value: value as Any)
@@ -3134,8 +3149,11 @@ extension Objects.Article: Decodable {
     savedByViewer = map["savedByViewer"]
     shareInfo = map["shareInfo"]
     sharedComment = map["sharedComment"]
+    siteIcon = map["siteIcon"]
+    siteName = map["siteName"]
     slug = map["slug"]
     title = map["title"]
+    uploadFileId = map["uploadFileId"]
     url = map["url"]
   }
 }
@@ -3587,6 +3605,51 @@ extension Fields where TypeLock == Objects.Article {
       return selection.mock()
     }
   }
+
+  func uploadFileId() throws -> String? {
+    let field = GraphQLField.leaf(
+      name: "uploadFileId",
+      arguments: []
+    )
+    select(field)
+
+    switch response {
+    case let .decoding(data):
+      return data.uploadFileId[field.alias!]
+    case .mocking:
+      return nil
+    }
+  }
+
+  func siteName() throws -> String? {
+    let field = GraphQLField.leaf(
+      name: "siteName",
+      arguments: []
+    )
+    select(field)
+
+    switch response {
+    case let .decoding(data):
+      return data.siteName[field.alias!]
+    case .mocking:
+      return nil
+    }
+  }
+
+  func siteIcon() throws -> String? {
+    let field = GraphQLField.leaf(
+      name: "siteIcon",
+      arguments: []
+    )
+    select(field)
+
+    switch response {
+    case let .decoding(data):
+      return data.siteIcon[field.alias!]
+    case .mocking:
+      return nil
+    }
+  }
 }
 
 extension Selection where TypeLock == Never, Type == Never {
@@ -10784,7 +10847,7 @@ extension Fields where TypeLock == Objects.Label {
     }
   }
 
-  func createdAt() throws -> DateTime {
+  func createdAt() throws -> DateTime? {
     let field = GraphQLField.leaf(
       name: "createdAt",
       arguments: []
@@ -10793,12 +10856,9 @@ extension Fields where TypeLock == Objects.Label {
 
     switch response {
     case let .decoding(data):
-      if let data = data.createdAt[field.alias!] {
-        return data
-      }
-      throw HttpError.badpayload
+      return data.createdAt[field.alias!]
     case .mocking:
-      return DateTime.mockValue
+      return nil
     }
   }
 }
@@ -16779,6 +16839,10 @@ extension Enums {
   /// SortBy
   enum SortBy: String, CaseIterable, Codable {
     case updatedTime = "UPDATED_TIME"
+
+    case score = "SCORE"
+
+    case savedAt = "SAVED_AT"
   }
 }
 
@@ -16956,6 +17020,8 @@ extension Enums {
     case payloadTooLarge = "PAYLOAD_TOO_LARGE"
 
     case uploadFileMissing = "UPLOAD_FILE_MISSING"
+
+    case elasticError = "ELASTIC_ERROR"
   }
 }
 

apple/OmnivoreKit/Sources/Services/DataService/Mutations/CreateLabelPublisher.swift

@@ -0,0 +1,62 @@
+import Combine
+import Foundation
+import Models
+import SwiftGraphQL
+
+public extension DataService {
+  func createLabelPublisher(
+    name: String,
+    color: String,
+    description: String?
+  ) -> AnyPublisher<FeedItemLabel, BasicError> {
+    enum MutationResult {
+      case saved(label: FeedItemLabel)
+      case error(errorCode: Enums.CreateLabelErrorCode)
+    }
+
+    let selection = Selection<MutationResult, Unions.CreateLabelResult> {
+      try $0.on(
+        createLabelSuccess: .init { .saved(label: try $0.label(selection: feedItemLabelSelection)) },
+        createLabelError: .init { .error(errorCode: try $0.errorCodes().first ?? .badRequest) }
+      )
+    }
+
+    let mutation = Selection.Mutation {
+      try $0.createLabel(
+        input: InputObjects.CreateLabelInput(
+          name: name,
+          color: color,
+          description: OptionalArgument(description)
+        ),
+        selection: selection
+      )
+    }
+
+    let path = appEnvironment.graphqlPath
+    let headers = networker.defaultHeaders
+
+    return Deferred {
+      Future { promise in
+        send(mutation, to: path, headers: headers) { result in
+          switch result {
+          case let .success(payload):
+            if let graphqlError = payload.errors {
+              promise(.failure(.message(messageText: "graphql error: \(graphqlError)")))
+            }
+
+            switch payload.data {
+            case let .saved(label: label):
+              promise(.success(label))
+            case let .error(errorCode: errorCode):
+              promise(.failure(.message(messageText: errorCode.rawValue)))
+            }
+          case .failure:
+            promise(.failure(.message(messageText: "graphql error")))
+          }
+        }
+      }
+    }
+    .receive(on: DispatchQueue.main)
+    .eraseToAnyPublisher()
+  }
+}

apple/OmnivoreKit/Sources/Services/DataService/Mutations/RemoveLabelPublisher.swift

@@ -0,0 +1,53 @@
+import Combine
+import Foundation
+import Models
+import SwiftGraphQL
+
+public extension DataService {
+  func removeLabelPublisher(labelID: String) -> AnyPublisher<Bool, BasicError> {
+    enum MutationResult {
+      case success(labelID: String)
+      case error(errorCode: Enums.DeleteLabelErrorCode)
+    }
+
+    let selection = Selection<MutationResult, Unions.DeleteLabelResult> {
+      try $0.on(
+        deleteLabelSuccess: .init {
+          .success(labelID: try $0.label(selection: Selection.Label { try $0.id() }))
+        },
+        deleteLabelError: .init { .error(errorCode: try $0.errorCodes().first ?? .badRequest) }
+      )
+    }
+
+    let mutation = Selection.Mutation {
+      try $0.deleteLabel(id: labelID, selection: selection)
+    }
+
+    let path = appEnvironment.graphqlPath
+    let headers = networker.defaultHeaders
+
+    return Deferred {
+      Future { promise in
+        send(mutation, to: path, headers: headers) { result in
+          switch result {
+          case let .success(payload):
+            if payload.errors != nil {
+              promise(.failure(.message(messageText: "Error removing label")))
+            }
+
+            switch payload.data {
+            case .success:
+              promise(.success(true))
+            case .error:
+              promise(.failure(.message(messageText: "Error removing label")))
+            }
+          case .failure:
+            promise(.failure(.message(messageText: "Error removing label")))
+          }
+        }
+      }
+    }
+    .receive(on: DispatchQueue.main)
+    .eraseToAnyPublisher()
+  }
+}

apple/OmnivoreKit/Sources/Services/DataService/Mutations/UpdateArticleLabelsPublisher.swift

@@ -0,0 +1,57 @@
+import Combine
+import Foundation
+import Models
+import SwiftGraphQL
+
+public extension DataService {
+  func updateArticleLabelsPublisher(itemID: String, labelIDs: [String]) -> AnyPublisher<[FeedItemLabel], BasicError> {
+    enum MutationResult {
+      case saved(feedItem: [FeedItemLabel])
+      case error(errorCode: Enums.SetLabelsErrorCode)
+    }
+
+    let selection = Selection<MutationResult, Unions.SetLabelsResult> {
+      try $0.on(
+        setLabelsSuccess: .init { .saved(feedItem: try $0.labels(selection: feedItemLabelSelection.list)) },
+        setLabelsError: .init { .error(errorCode: try $0.errorCodes().first ?? .badRequest) }
+      )
+    }
+
+    let mutation = Selection.Mutation {
+      try $0.setLabels(
+        input: InputObjects.SetLabelsInput(
+          linkId: itemID,
+          labelIds: labelIDs
+        ),
+        selection: selection
+      )
+    }
+
+    let path = appEnvironment.graphqlPath
+    let headers = networker.defaultHeaders
+
+    return Deferred {
+      Future { promise in
+        send(mutation, to: path, headers: headers) { result in
+          switch result {
+          case let .success(payload):
+            if let graphqlError = payload.errors {
+              promise(.failure(.message(messageText: graphqlError.first.debugDescription)))
+            }
+
+            switch payload.data {
+            case let .saved(labels):
+              promise(.success(labels))
+            case .error:
+              promise(.failure(.message(messageText: "failed to set labels")))
+            }
+          case .failure:
+            promise(.failure(.message(messageText: "failed to set labels")))
+          }
+        }
+      }
+    }
+    .receive(on: DispatchQueue.main)
+    .eraseToAnyPublisher()
+  }
+}

apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift

@@ -3,7 +3,6 @@ import Foundation
 import Models
 import SwiftGraphQL
 
-// swiftlint:disable:next function_body_length
 public extension DataService {
   func articleContentPublisher(username: String, slug: String) -> AnyPublisher<ArticleContent, ServerError> {
     enum QueryResult {

apple/OmnivoreKit/Sources/Services/DataService/Queries/LabelsPublisher.swift

@@ -0,0 +1,49 @@
+import Combine
+import Foundation
+import Models
+import SwiftGraphQL
+
+public extension DataService {
+  func labelsPublisher() -> AnyPublisher<[FeedItemLabel], ServerError> {
+    enum QueryResult {
+      case success(result: [FeedItemLabel])
+      case error(error: String)
+    }
+
+    let selection = Selection<QueryResult, Unions.LabelsResult> {
+      try $0.on(labelsSuccess: .init {
+        QueryResult.success(result: try $0.labels(selection: feedItemLabelSelection.list))
+      },
+      labelsError: .init {
+        QueryResult.error(error: try $0.errorCodes().description)
+      })
+    }
+
+    let query = Selection.Query {
+      try $0.labels(selection: selection)
+    }
+
+    let path = appEnvironment.graphqlPath
+    let headers = networker.defaultHeaders
+
+    return Deferred {
+      Future { promise in
+        send(query, to: path, headers: headers) { result in
+          switch result {
+          case let .success(payload):
+            switch payload.data {
+            case let .success(result: result):
+              promise(.success(result))
+            case .error:
+              promise(.failure(.unknown))
+            }
+          case .failure:
+            promise(.failure(.unknown))
+          }
+        }
+      }
+    }
+    .receive(on: DispatchQueue.main)
+    .eraseToAnyPublisher()
+  }
+}

apple/OmnivoreKit/Sources/Services/DataService/Queries/LibraryItemsQuery.swift

@@ -143,7 +143,8 @@ let homeFeedItemSelection = Selection.Article {
     publishDate: try $0.publishedAt()?.value,
     slug: try $0.slug(),
     isArchived: try $0.isArchived(),
-    contentReader: try $0.contentReader().rawValue
+    contentReader: try $0.contentReader().rawValue,
+    labels: try $0.labels(selection: feedItemLabelSelection.list.nullable) ?? []
   )
 }
 

apple/OmnivoreKit/Sources/Services/DataService/Selections/FeedItemLabelSelection.swift

@@ -0,0 +1,12 @@
+import Models
+import SwiftGraphQL
+
+let feedItemLabelSelection = Selection.Label {
+  FeedItemLabel(
+    id: try $0.id(),
+    name: try $0.name(),
+    color: try $0.color(),
+    createdAt: try $0.createdAt()?.value,
+    description: try $0.description()
+  )
+}

apple/OmnivoreKit/Sources/Utils/ColorUtils.swift

@@ -0,0 +1,63 @@
+import SwiftUI
+
+public extension Color {
+  /// Inititializes a `Color` from a hex value
+  /// - Parameter hex: Color hex value. ex: `#FFFFFF`
+  ///
+  init?(hex: String) {
+    var hexSanitized = hex.trimmingCharacters(in: .whitespacesAndNewlines)
+    hexSanitized = hexSanitized.replacingOccurrences(of: "#", with: "")
+
+    var rgb: UInt64 = 0
+
+    var red: CGFloat = 0.0
+    var green: CGFloat = 0.0
+    var blue: CGFloat = 0.0
+    var alpha: CGFloat = 1.0
+
+    let length = hexSanitized.count
+
+    guard Scanner(string: hexSanitized).scanHexInt64(&rgb) else { return nil }
+
+    if length == 6 {
+      red = CGFloat((rgb & 0xFF0000) >> 16) / 255.0
+      green = CGFloat((rgb & 0x00FF00) >> 8) / 255.0
+      blue = CGFloat(rgb & 0x0000FF) / 255.0
+    } else if length == 8 {
+      red = CGFloat((rgb & 0xFF00_0000) >> 24) / 255.0
+      green = CGFloat((rgb & 0x00FF_0000) >> 16) / 255.0
+      blue = CGFloat((rgb & 0x0000_FF00) >> 8) / 255.0
+      alpha = CGFloat(rgb & 0x0000_00FF) / 255.0
+
+    } else {
+      return nil
+    }
+
+    self.init(red: red, green: green, blue: blue, opacity: alpha)
+  }
+
+  var hex: String? {
+    if let hexValue = toHex() {
+      return "#\(hexValue)"
+    } else {
+      return nil
+    }
+  }
+
+  private func toHex() -> String? {
+    let uic = UIColor(self)
+    guard let components = uic.cgColor.components, components.count >= 3 else {
+      return nil
+    }
+    let red = Float(components[0])
+    let green = Float(components[1])
+    let blue = Float(components[2])
+
+    return String(
+      format: "%02lX%02lX%02lX",
+      lroundf(red * 255),
+      lroundf(green * 255),
+      lroundf(blue * 255)
+    )
+  }
+}

apple/OmnivoreKit/Sources/Utils/FeatureFlags.swift

@@ -14,6 +14,6 @@ public enum FeatureFlag {
   public static let enablePushNotifications = false
   public static let enableShareButton = false
   public static let enableSnooze = false
-  public static let showFeedItemTags = false
+  public static let enableLabels = true
   public static let useLocalWebView = true
 }

apple/OmnivoreKit/Sources/Views/FeedItem/GridCard.swift

@@ -5,6 +5,7 @@ import Utils
 public enum GridCardAction {
   case toggleArchiveStatus
   case delete
+  case editLabels
 }
 
 public struct GridCard: View {
@@ -42,6 +43,10 @@ public struct GridCard: View {
 
   var contextMenuView: some View {
     Group {
+      Button(
+        action: { menuActionHandler(.editLabels) },
+        label: { Label("Edit Labels", systemImage: "tag") }
+      )
       Button(
         action: { menuActionHandler(.toggleArchiveStatus) },
         label: {
@@ -156,11 +161,12 @@ public struct GridCard: View {
         .onTapGesture { tapHandler() }
 
         // Category Labels
-        if FeatureFlag.showFeedItemTags {
+        if FeatureFlag.enableLabels {
           ScrollView(.horizontal, showsIndicators: false) {
             HStack {
-              TextChip(text: "label", color: .red)
-              TextChip(text: "longer label", color: .blue)
+              ForEach(item.labels, id: \.self) {
+                TextChip(feedItemLabel: $0)
+              }
               Spacer()
             }
             .frame(height: 30)

apple/OmnivoreKit/Sources/Views/TextChip.swift

@@ -1,11 +1,25 @@
+import Models
 import SwiftUI
+import Utils
+
+public struct TextChip: View {
+  public init(text: String, color: Color) {
+    self.text = text
+    self.color = color
+  }
+
+  public init?(feedItemLabel: FeedItemLabel) {
+    guard let color = Color(hex: feedItemLabel.color) else { return nil }
+
+    self.text = feedItemLabel.name
+    self.color = color
+  }
 
-struct TextChip: View {
   let text: String
   let color: Color
   let cornerRadius = 20.0
 
-  var body: some View {
+  public var body: some View {
     Text(text)
       .padding(.horizontal, 10)
       .padding(.vertical, 5)

docker-compose.yml

@@ -84,6 +84,7 @@ services:
       - PG_PORT=5432
       - PG_POOL_MAX=20
       - ELASTIC_URL=http://elastic:9200
+      - JAEGER_HOST=jaeger
       - IMAGE_PROXY_URL=http://localhost:9999
       - IMAGE_PROXY_SECRET=some-secret
       - JWT_SECRET=some_secret

package.json

@@ -8,7 +8,7 @@
   ],
   "license": "UNLICENSED",
   "scripts": {
-    "test": "lerna run test --ignore @omnivore/web",
+    "test": "lerna run --no-bail test --ignore @omnivore/web",
     "lint": "lerna run lint --ignore @omnivore/web",
     "build": "lerna run build --ignore @omnivore/web",
     "bootstrap": "lerna bootstrap",

packages/api/.env.test

@@ -27,3 +27,5 @@ PREVIEW_IMAGE_WRAPPER_ID='selected_highlight_wrapper'
 SEGMENT_WRITE_KEY='test'
 REMINDER_TASK_HANDLER_URL=http://localhost:4000/svc/reminders/trigger
 PUBSUB_VERIFICATION_TOKEN='123456'
+PUPPETEER_TASK_HANDLER_URL=http://localhost:9090/
+

packages/api/Dockerfile

@@ -2,7 +2,6 @@ FROM node:14.18-alpine as builder
 
 WORKDIR /app
 
-ENV NODE_ENV production
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
 
 COPY package.json .
@@ -14,7 +13,7 @@ COPY .eslintrc .
 COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
 COPY /packages/api/package.json ./packages/api/package.json
 
-RUN yarn install --pure-lockfile --production
+RUN yarn install --pure-lockfile
 
 ADD /packages/readabilityjs ./packages/readabilityjs
 ADD /packages/api ./packages/api
@@ -22,7 +21,10 @@ ADD /packages/api ./packages/api
 RUN yarn
 RUN yarn workspace @omnivore/api build
 
-
+# After building, fetch the production dependencies
+RUN rm -rf /app/packages/api/node_modules
+RUN rm -rf /app/node_modules
+RUN yarn install --pure-lockfile --production
 
 FROM node:14.18-alpine as runner
 
@@ -38,7 +40,7 @@ COPY --from=builder /app/packages/api/package.json /app/packages/api/package.jso
 COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules
 COPY --from=builder /app/node_modules /app/node_modules
 COPY --from=builder /app/package.json /app/package.json
-
+COPY --from=builder /app/packages/api/index_settings.json /app/packages/api/index_settings.json
 EXPOSE 8080
 
 CMD ["yarn", "workspace", "@omnivore/api", "start"]

packages/api/package.json

@@ -30,32 +30,12 @@
     "@opentelemetry/instrumentation-pg": "^0.24.0",
     "@opentelemetry/node": "^0.24.0",
     "@opentelemetry/resources": "^0.24.0",
-    "@opentelemetry/semantic-conventions": "^0.24.0",
+    "@opentelemetry/semantic-conventions": "^1.0.1",
     "@opentelemetry/tracing": "^0.24.0",
     "@sendgrid/mail": "^7.6.0",
     "@sentry/integrations": "^6.19.1",
     "@sentry/node": "^5.26.0",
     "@sentry/tracing": "^5.26.0",
-    "@types/analytics-node": "^3.1.7",
-    "@types/bcryptjs": "^2.4.2",
-    "@types/chai": "^4.2.18",
-    "@types/chai-string": "^1.4.2",
-    "@types/cookie": "^0.4.0",
-    "@types/cookie-parser": "^1.4.2",
-    "@types/dompurify": "^2.0.4",
-    "@types/express": "^4.17.7",
-    "@types/highlightjs": "^9.12.2",
-    "@types/intercom-client": "^2.11.8",
-    "@types/jsdom": "^16.2.3",
-    "@types/jsonwebtoken": "^8.5.0",
-    "@types/luxon": "^1.25.0",
-    "@types/mocha": "^8.2.2",
-    "@types/oauth": "^0.9.1",
-    "@types/sanitize-html": "^1.27.1",
-    "@types/supertest": "^2.0.11",
-    "@types/urlsafe-base64": "^1.0.28",
-    "@types/uuid": "^8.3.0",
-    "@types/voca": "^1.4.0",
     "analytics-node": "^6.0.0",
     "apollo-datasource": "^3.3.1",
     "apollo-server-express": "^3.6.3",
@@ -82,8 +62,9 @@
     "jwks-rsa": "^2.0.3",
     "knex": "0.21.12",
     "knex-stringcase": "^1.4.2",
-    "luxon": "^1.25.0",
+    "luxon": "^2.3.1",
     "nanoid": "^3.1.25",
+    "nodemailer": "^6.7.3",
     "normalize-url": "^6.1.0",
     "oauth": "^0.9.15",
     "pg": "^8.3.3",
@@ -94,8 +75,8 @@
     "snake-case": "^3.0.3",
     "supertest": "^6.2.2",
     "ts-loader": "^8.0.3",
-    "typeorm": "^0.2.37",
-    "typeorm-naming-strategies": "^2.0.0",
+    "typeorm": "^0.3.4",
+    "typeorm-naming-strategies": "^4.1.0",
     "urlsafe-base64": "^1.0.0",
     "uuid": "^8.3.1",
     "voca": "^1.4.0",
@@ -104,10 +85,29 @@
   "devDependencies": {
     "@babel/register": "^7.14.5",
     "@istanbuljs/nyc-config-typescript": "^1.0.2",
-    "@types/analytics-node": "^3.1.7",
     "@types/highlightjs": "^9.12.2",
     "@types/nanoid": "^3.0.0",
     "@types/private-ip": "^1.0.0",
+    "@types/analytics-node": "^3.1.7",
+    "@types/bcryptjs": "^2.4.2",
+    "@types/chai": "^4.2.18",
+    "@types/chai-string": "^1.4.2",
+    "@types/cookie": "^0.4.0",
+    "@types/cookie-parser": "^1.4.2",
+    "@types/dompurify": "^2.0.4",
+    "@types/express": "^4.17.7",
+    "@types/intercom-client": "^2.11.8",
+    "@types/jsdom": "^16.2.3",
+    "@types/jsonwebtoken": "^8.5.0",
+    "@types/luxon": "^1.25.0",
+    "@types/mocha": "^8.2.2",
+    "@types/nodemailer": "^6.4.4",
+    "@types/oauth": "^0.9.1",
+    "@types/sanitize-html": "^1.27.1",
+    "@types/supertest": "^2.0.11",
+    "@types/urlsafe-base64": "^1.0.28",
+    "@types/uuid": "^8.3.0",
+    "@types/voca": "^1.4.0",
     "chai": "^4.3.4",
     "chai-string": "^1.5.0",
     "circular-dependency-plugin": "^5.2.0",

packages/api/src/entity/follower.ts

@@ -1,16 +1,15 @@
 import {
-  Entity,
-  BaseEntity,
-  PrimaryGeneratedColumn,
   CreateDateColumn,
-  OneToOne,
+  Entity,
   JoinColumn,
+  OneToOne,
+  PrimaryGeneratedColumn,
 } from 'typeorm'
 
 import { User } from './user'
 
 @Entity({ name: 'user_friends' })
-export class Follower extends BaseEntity {
+export class Follower {
   @PrimaryGeneratedColumn('uuid')
   id?: string
 

packages/api/src/entity/groups/group.ts

@@ -1,18 +1,17 @@
 import {
-  Entity,
-  BaseEntity,
   Column,
-  PrimaryGeneratedColumn,
   CreateDateColumn,
-  UpdateDateColumn,
-  OneToOne,
+  Entity,
   JoinColumn,
+  OneToOne,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
 } from 'typeorm'
 
 import { User } from '../user'
 
 @Entity()
-export class Group extends BaseEntity {
+export class Group {
   @PrimaryGeneratedColumn('uuid')
   id?: string
 

packages/api/src/entity/groups/group_membership.ts

@@ -1,11 +1,10 @@
 import {
-  Entity,
-  BaseEntity,
-  PrimaryGeneratedColumn,
   CreateDateColumn,
-  UpdateDateColumn,
-  OneToOne,
+  Entity,
   JoinColumn,
+  OneToOne,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
 } from 'typeorm'
 
 import { User } from '../user'
@@ -13,7 +12,7 @@ import { Group } from './group'
 import { Invite } from './invite'
 
 @Entity()
-export class GroupMembership extends BaseEntity {
+export class GroupMembership {
   @PrimaryGeneratedColumn('uuid')
   id?: string
 

packages/api/src/entity/groups/invite.ts

@@ -1,19 +1,18 @@
 import {
-  Entity,
-  BaseEntity,
   Column,
-  PrimaryGeneratedColumn,
   CreateDateColumn,
-  UpdateDateColumn,
-  OneToOne,
+  Entity,
   JoinColumn,
+  OneToOne,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
 } from 'typeorm'
 
 import { User } from '../user'
 import { Group } from './group'
 
 @Entity()
-export class Invite extends BaseEntity {
+export class Invite {
   @PrimaryGeneratedColumn('uuid')
   id?: string
 

packages/api/src/entity/highlight.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -12,7 +11,7 @@ import { User } from './user'
 import { Page } from './page'
 
 @Entity({ name: 'highlight' })
-export class Highlight extends BaseEntity {
+export class Highlight {
   @PrimaryGeneratedColumn('uuid')
   id?: string
 

packages/api/src/entity/label.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -10,7 +9,7 @@ import {
 import { User } from './user'
 
 @Entity({ name: 'labels' })
-export class Label extends BaseEntity {
+export class Label {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/link.ts

@@ -10,7 +10,6 @@
 // shared_with_highlights                | boolean                  |           |          | false
 
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -27,7 +26,7 @@ import { Page } from './page'
 import { Label } from './label'
 
 @Entity({ name: 'links' })
-export class Link extends BaseEntity {
+export class Link {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/link_label.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   CreateDateColumn,
   Entity,
   JoinColumn,
@@ -10,7 +9,7 @@ import { Link } from './link'
 import { Label } from './label'
 
 @Entity({ name: 'link_labels' })
-export class LinkLabel extends BaseEntity {
+export class LinkLabel {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/newsletter_email.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -11,7 +10,7 @@ import {
 import { User } from './user'
 
 @Entity({ name: 'newsletter_emails' })
-export class NewsletterEmail extends BaseEntity {
+export class NewsletterEmail {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/page.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -8,7 +7,7 @@ import {
 } from 'typeorm'
 
 @Entity({ name: 'pages' })
-export class Page extends BaseEntity {
+export class Page {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/profile.ts

@@ -1,18 +1,17 @@
 import {
-  Entity,
-  BaseEntity,
   Column,
-  PrimaryGeneratedColumn,
   CreateDateColumn,
-  UpdateDateColumn,
-  OneToOne,
+  Entity,
   JoinColumn,
+  OneToOne,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
 } from 'typeorm'
 
 import { User } from './user'
 
 @Entity({ name: 'user_profile' })
-export class Profile extends BaseEntity {
+export class Profile {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/reminder.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -11,7 +10,7 @@ import {
 import { User } from './user'
 
 @Entity({ name: 'reminders' })
-export class Reminder extends BaseEntity {
+export class Reminder {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/reports/abuse_report.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -9,7 +8,7 @@ import {
 import { ReportType } from '../../generated/graphql'
 
 @Entity()
-export class AbuseReport extends BaseEntity {
+export class AbuseReport {
   @PrimaryGeneratedColumn('uuid')
   id?: string
 

packages/api/src/entity/reports/content_display_report.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -8,7 +7,7 @@ import {
 } from 'typeorm'
 
 @Entity()
-export class ContentDisplayReport extends BaseEntity {
+export class ContentDisplayReport {
   @PrimaryGeneratedColumn('uuid')
   id?: string
 

packages/api/src/entity/upload_file.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -11,7 +10,7 @@ import {
 import { User } from './user'
 
 @Entity({ name: 'upload_files' })
-export class UploadFile extends BaseEntity {
+export class UploadFile {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/user.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -14,7 +13,7 @@ import { Profile } from './profile'
 import { Label } from './label'
 
 @Entity()
-export class User extends BaseEntity {
+export class User {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/user_device_tokens.ts

@@ -1,5 +1,4 @@
 import {
-  BaseEntity,
   Column,
   CreateDateColumn,
   Entity,
@@ -10,7 +9,7 @@ import {
 import { User } from './user'
 
 @Entity({ name: 'user_device_tokens' })
-export class UserDeviceToken extends BaseEntity {
+export class UserDeviceToken {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 

packages/api/src/entity/utils.ts

@@ -1,4 +1,5 @@
-import { EntityManager } from 'typeorm'
+import { EntityManager, EntityTarget, Repository } from 'typeorm'
+import { AppDataSource } from '../server'
 
 export const setClaims = async (
   t: EntityManager,
@@ -9,3 +10,7 @@ export const setClaims = async (
     .query('SELECT * from omnivore.set_claims($1, $2)', [uid, dbRole])
     .then()
 }
+
+export const getRepository = <T>(entity: EntityTarget<T>): Repository<T> => {
+  return AppDataSource.getRepository(entity)
+}

packages/api/src/events/user/user_created.ts

@@ -32,8 +32,7 @@ export class FollowOmnivoreUser implements EntitySubscriberInterface<Profile> {
 
     await event.manager
       .getRepository(Follower)
-      .create({ user: event.entity.user, followee: omnivoreProfile.user })
-      .save()
+      .save({ user: event.entity.user, followee: omnivoreProfile.user })
 
     await event.manager.query(
       `insert into omnivore.links (user_id, article_id, article_url, article_hash, slug)

packages/api/src/resolvers/labels/index.ts

@@ -24,10 +24,11 @@ import { analytics } from '../../utils/analytics'
 import { env } from '../../env'
 import { User } from '../../entity/user'
 import { Label } from '../../entity/label'
-import { getManager, getRepository, ILike } from 'typeorm'
-import { setClaims } from '../../entity/utils'
+import { ILike, In } from 'typeorm'
+import { getRepository, setClaims } from '../../entity/utils'
 import { deleteLabelInPages, getPageById, updatePage } from '../../elastic'
 import { createPubSubClient } from '../../datalayer/pubsub'
+import { AppDataSource } from '../../server'
 
 export const labelsResolver = authorized<LabelsSuccess, LabelsError>(
   async (_obj, _params, { claims: { uid }, log }) => {
@@ -80,7 +81,7 @@ export const createLabelResolver = authorized<
   const { name, color, description } = input
 
   try {
-    const user = await getRepository(User).findOne(uid)
+    const user = await getRepository(User).findOneBy({ id: uid })
     if (!user) {
       return {
         errorCodes: [CreateLabelErrorCode.Unauthorized],
@@ -88,11 +89,9 @@ export const createLabelResolver = authorized<
     }
 
     // Check if label already exists ignoring case of name
-    const existingLabel = await getRepository(Label).findOne({
-      where: {
-        user,
-        name: ILike(name),
-      },
+    const existingLabel = await getRepository(Label).findOneBy({
+      user: { id: user.id },
+      name: ILike(name),
     })
     if (existingLabel) {
       return {
@@ -100,14 +99,12 @@ export const createLabelResolver = authorized<
       }
     }
 
-    const label = await getRepository(Label)
-      .create({
-        user,
-        name,
-        color,
-        description: description || '',
-      })
-      .save()
+    const label = await getRepository(Label).save({
+      user,
+      name,
+      color,
+      description: description || '',
+    })
 
     analytics.track({
       userId: uid,
@@ -207,14 +204,15 @@ export const deleteLabelResolver = authorized<
   log.info('deleteLabelResolver')
 
   try {
-    const user = await getRepository(User).findOne(uid)
+    const user = await getRepository(User).findOneBy({ id: uid })
     if (!user) {
       return {
         errorCodes: [DeleteLabelErrorCode.Unauthorized],
       }
     }
 
-    const label = await getRepository(Label).findOne(labelId, {
+    const label = await getRepository(Label).findOne({
+      where: { id: labelId },
       relations: ['user'],
     })
     if (!label) {
@@ -229,7 +227,7 @@ export const deleteLabelResolver = authorized<
       }
     }
 
-    const result = await getManager().transaction(async (t) => {
+    const result = await AppDataSource.transaction(async (t) => {
       await setClaims(t, uid)
       return t.getRepository(Label).delete(labelId)
     })
@@ -276,7 +274,7 @@ export const setLabelsResolver = authorized<
   const { linkId: pageId, labelIds } = input
 
   try {
-    const user = await getRepository(User).findOne(uid)
+    const user = await getRepository(User).findOneBy({ id: uid })
     if (!user) {
       return {
         errorCodes: [SetLabelsErrorCode.Unauthorized],
@@ -290,10 +288,8 @@ export const setLabelsResolver = authorized<
       }
     }
 
-    const labels = await getRepository(Label).findByIds(labelIds, {
-      where: {
-        user,
-      },
+    const labels = await getRepository(Label).find({
+      where: { id: In(labelIds), user: { id: user.id } },
       relations: ['user'],
     })
     if (labels.length !== labelIds.length) {

packages/api/src/resolvers/newsletters/index.ts

@@ -1,15 +1,15 @@
 import { authorized } from '../../utils/helpers'
 import {
-  CreateNewsletterEmailSuccess,
   CreateNewsletterEmailError,
   CreateNewsletterEmailErrorCode,
-  NewsletterEmailsSuccess,
-  NewsletterEmailsError,
-  NewsletterEmailsErrorCode,
+  CreateNewsletterEmailSuccess,
+  DeleteNewsletterEmailError,
   DeleteNewsletterEmailErrorCode,
   DeleteNewsletterEmailSuccess,
-  DeleteNewsletterEmailError,
   MutationDeleteNewsletterEmailArgs,
+  NewsletterEmailsError,
+  NewsletterEmailsErrorCode,
+  NewsletterEmailsSuccess,
 } from '../../generated/graphql'
 import {
   createNewsletterEmail,
@@ -19,6 +19,8 @@ import {
 import { NewsletterEmail } from '../../entity/newsletter_email'
 import { analytics } from '../../utils/analytics'
 import { env } from '../../env'
+import { AppDataSource } from '../../server'
+import { User } from '../../entity/user'
 
 export const createNewsletterEmailResolver = authorized<
   CreateNewsletterEmailSuccess,
@@ -55,7 +57,16 @@ export const newsletterEmailsResolver = authorized<
   console.log('newsletterEmailsResolver')
 
   try {
-    const newsletterEmails = await getNewsletterEmails(claims.uid)
+    const user = await AppDataSource.getRepository(User).findOneBy({
+      id: claims.uid,
+    })
+    if (!user) {
+      return Promise.reject({
+        errorCode: NewsletterEmailsErrorCode.Unauthorized,
+      })
+    }
+
+    const newsletterEmails = await getNewsletterEmails(user.id)
 
     return {
       newsletterEmails: newsletterEmails,
@@ -84,10 +95,14 @@ export const deleteNewsletterEmailResolver = authorized<
   })
 
   try {
-    const newsletterEmail = await NewsletterEmail.findOne(
-      args.newsletterEmailId,
-      { relations: ['user'] }
-    )
+    const newsletterEmail = await AppDataSource.getRepository(
+      NewsletterEmail
+    ).findOne({
+      where: {
+        id: args.newsletterEmailId,
+      },
+      relations: ['user'],
+    })
 
     if (!newsletterEmail) {
       return {

packages/api/src/routers/svc/emails.ts

@@ -4,6 +4,7 @@ import { sendEmail } from '../../utils/sendEmail'
 import { analytics } from '../../utils/analytics'
 import { getNewsletterEmail } from '../../services/newsletters'
 import { env } from '../../env'
+import { v4 as uuid } from 'uuid'
 import { findNewsletterUrl, isProbablyNewsletter } from '../../utils/parser'
 import { saveNewsletterEmail } from '../../services/save_newsletter_email'
 
@@ -59,7 +60,7 @@ export function emailsServiceRouter() {
           author: data.from,
           url:
             (await findNewsletterUrl(data.html)) ||
-            'https://omnivore.app/no_url',
+            'https://omnivore.app/no_url?q' + uuid(),
         })
         res.status(200).send('Newsletter')
         return

packages/api/src/server.ts

@@ -19,7 +19,7 @@ import { articleRouter } from './routers/article_router'
 import { mobileAuthRouter } from './routers/auth/mobile/mobile_auth_router'
 import { contentServiceRouter } from './routers/svc/content'
 import { localDebugRouter } from './routers/local_debug_router'
-import { Connection, createConnection } from 'typeorm'
+import { DataSource } from 'typeorm'
 import { SnakeNamingStrategy } from 'typeorm-naming-strategies'
 import { linkServiceRouter } from './routers/svc/links'
 import UserModel from './datalayer/user'
@@ -57,21 +57,19 @@ export const initModels = (kx: Knex, cache = true): DataModels => ({
   reminder: new ReminderModel(kx, cache),
 })
 
-const initEntities = async (): Promise<Connection> => {
-  return createConnection({
-    type: 'postgres',
-    host: env.pg.host,
-    port: env.pg.port,
-    schema: 'omnivore',
-    username: env.pg.userName,
-    password: env.pg.password,
-    database: env.pg.dbName,
-    logging: ['query', 'info'],
-    entities: [__dirname + '/entity/**/*{.js,.ts}'],
-    subscribers: [__dirname + '/events/**/*{.js,.ts}'],
-    namingStrategy: new SnakeNamingStrategy(),
-  })
-}
+export const AppDataSource = new DataSource({
+  type: 'postgres',
+  host: env.pg.host,
+  port: env.pg.port,
+  schema: 'omnivore',
+  username: env.pg.userName,
+  password: env.pg.password,
+  database: env.pg.dbName,
+  logging: ['query', 'info'],
+  entities: [__dirname + '/entity/**/*{.js,.ts}'],
+  subscribers: [__dirname + '/events/**/*{.js,.ts}'],
+  namingStrategy: new SnakeNamingStrategy(),
+})
 
 export const createApp = (): {
   app: Express
@@ -126,7 +124,7 @@ const main = async (): Promise<void> => {
   // If creating the DB entities fails, we want this to throw
   // so the container will be restarted and not come online
   // as healthy.
-  await initEntities()
+  await AppDataSource.initialize()
 
   await initElasticsearch()
 

packages/api/src/services/archive_link.ts

@@ -1,13 +1,13 @@
-import { getManager } from 'typeorm'
 import { Link } from '../entity/link'
 import { setClaims } from '../entity/utils'
+import { AppDataSource } from '../server'
 
 export const setLinkArchived = async (
   userId: string,
   linkId: string,
   archived: boolean
 ): Promise<void> => {
-  await getManager().transaction(async (t) => {
+  await AppDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     await t.getRepository(Link).update(
       {

packages/api/src/services/create_group.ts

@@ -1,9 +1,9 @@
-import { getManager } from 'typeorm'
 import { User } from '../entity/user'
 import { Group } from '../entity/groups/group'
 import { Invite } from '../entity/groups/invite'
 import { GroupMembership } from '../entity/groups/group_membership'
 import { nanoid } from 'nanoid'
+import { AppDataSource } from '../server'
 
 export const createGroup = async (input: {
   admin: User
@@ -11,15 +11,12 @@ export const createGroup = async (input: {
   maxMembers?: number
   expiresInDays?: number
 }): Promise<[Group, Invite]> => {
-  const [group, invite] = await getManager().transaction<[Group, Invite]>(
+  const [group, invite] = await AppDataSource.transaction<[Group, Invite]>(
     async (t) => {
-      const group = await t
-        .getRepository(Group)
-        .create({
-          name: input.name,
-          createdBy: input.admin,
-        })
-        .save()
+      const group = await t.getRepository(Group).save({
+        name: input.name,
+        createdBy: input.admin,
+      })
 
       const code = nanoid(8)
       const expirationTime = (() => {
@@ -27,25 +24,19 @@ export const createGroup = async (input: {
         r.setDate(r.getDate() + (input.expiresInDays || 7))
         return r
       })()
-      const invite = await t
-        .getRepository(Invite)
-        .create({
-          group,
-          code,
-          createdBy: input.admin,
-          maxMembers: input.maxMembers || 50,
-          expirationTime: expirationTime,
-        })
-        .save()
+      const invite = await t.getRepository(Invite).save({
+        group,
+        code,
+        createdBy: input.admin,
+        maxMembers: input.maxMembers || 50,
+        expirationTime: expirationTime,
+      })
       // Add the admin to the group as its first user
-      await t
-        .getRepository(GroupMembership)
-        .create({
-          user: input.admin,
-          group,
-          invite,
-        })
-        .save()
+      await t.getRepository(GroupMembership).save({
+        user: input.admin,
+        group,
+        invite,
+      })
       return [group, invite]
     }
   )

packages/api/src/services/create_user.ts

@@ -1,12 +1,14 @@
 import { AuthProvider } from '../routers/auth/auth_types'
 import { MembershipTier } from '../datalayer/user/model'
-import { EntityManager, getManager, getRepository } from 'typeorm'
+import { EntityManager } from 'typeorm'
 import { User } from '../entity/user'
 import { Profile } from '../entity/profile'
 import { SignupErrorCode } from '../generated/graphql'
 import { validateUsername } from '../utils/usernamePolicy'
 import { Invite } from '../entity/groups/invite'
 import { GroupMembership } from '../entity/groups/group_membership'
+import { AppDataSource } from '../server'
+import { getRepository } from '../entity/utils'
 
 export const createUser = async (input: {
   provider: AuthProvider
@@ -28,15 +30,12 @@ export const createUser = async (input: {
     }
 
     // create profile if user exists but profile does not exist
-    const profile = await getManager()
-      .getRepository(Profile)
-      .create({
-        username: input.username,
-        pictureUrl: input.pictureUrl,
-        bio: input.bio,
-        user: existingUser,
-      })
-      .save()
+    const profile = await getRepository(Profile).save({
+      username: input.username,
+      pictureUrl: input.pictureUrl,
+      bio: input.bio,
+      user: existingUser,
+    })
 
     return [existingUser, profile]
   }
@@ -45,10 +44,10 @@ export const createUser = async (input: {
     return Promise.reject({ errorCode: SignupErrorCode.InvalidUsername })
   }
 
-  const [user, profile] = await getManager().transaction<[User, Profile]>(
+  const [user, profile] = await AppDataSource.transaction<[User, Profile]>(
     async (t) => {
       let hasInvite = false
-      let invite: Invite | undefined = undefined
+      let invite: Invite | null = null
 
       if (input.inviteCode) {
         const inviteCodeRepo = t.getRepository(Invite)
@@ -60,37 +59,28 @@ export const createUser = async (input: {
           hasInvite = true
         }
       }
-      const user = await t
-        .getRepository(User)
-        .create({
-          source: input.provider,
-          membership:
-            input.membershipTier ||
-            (hasInvite ? MembershipTier.Beta : MembershipTier.WaitList),
-          name: input.name,
-          email: input.email,
-          sourceUserId: input.sourceUserId,
-          password: input.password,
-        })
-        .save()
-      const profile = await t
-        .getRepository(Profile)
-        .create({
-          username: input.username,
-          pictureUrl: input.pictureUrl,
-          bio: input.bio,
-          user,
-        })
-        .save()
+      const user = await t.getRepository(User).save({
+        source: input.provider,
+        membership:
+          input.membershipTier ||
+          (hasInvite ? MembershipTier.Beta : MembershipTier.WaitList),
+        name: input.name,
+        email: input.email,
+        sourceUserId: input.sourceUserId,
+        password: input.password,
+      })
+      const profile = await t.getRepository(Profile).save({
+        username: input.username,
+        pictureUrl: input.pictureUrl,
+        bio: input.bio,
+        user,
+      })
       if (hasInvite && invite) {
-        await t
-          .getRepository(GroupMembership)
-          .create({
-            user: user,
-            invite: invite,
-            group: invite.group,
-          })
-          .save()
+        await t.getRepository(GroupMembership).save({
+          user: user,
+          invite: invite,
+          group: invite.group,
+        })
       }
       return [user, profile]
     }
@@ -119,7 +109,7 @@ const validateInvite = async (
   return true
 }
 
-const getUser = async (email: string): Promise<User | undefined> => {
+const getUser = async (email: string): Promise<User | null> => {
   const userRepo = getRepository(User)
 
   return userRepo.findOne({

packages/api/src/services/followers.ts

@@ -1,6 +1,6 @@
-import { getRepository } from 'typeorm'
 import { User } from '../entity/user'
 import { Follower } from '../entity/follower'
+import { getRepository } from '../entity/utils'
 
 export const getUserFollowers = async (
   user: User,
@@ -9,7 +9,7 @@ export const getUserFollowers = async (
 ): Promise<User[]> => {
   return (
     await getRepository(Follower).find({
-      where: { user: user },
+      where: { user: { id: user.id } },
       relations: ['user', 'followee'],
       skip: offset,
       take: count,
@@ -24,7 +24,7 @@ export const getUserFollowing = async (
 ): Promise<User[]> => {
   return (
     await getRepository(Follower).find({
-      where: { followee: user },
+      where: { followee: { id: user.id } },
       relations: ['user', 'followee'],
       skip: offset,
       take: count,

packages/api/src/services/labels.ts

@@ -1,10 +1,11 @@
-import DataLoader from 'dataloader'
 import { Label } from '../entity/label'
-import { getRepository, ILike, In } from 'typeorm'
-import { Link } from '../entity/link'
+import { ILike, In } from 'typeorm'
 import { PageContext } from '../elastic/types'
 import { User } from '../entity/user'
 import { addLabelInPage } from '../elastic'
+import { getRepository } from '../entity/utils'
+import { Link } from '../entity/link'
+import DataLoader from 'dataloader'
 
 const batchGetLabelsFromLinkIds = async (
   linkIds: readonly string[]
@@ -30,13 +31,16 @@ export const addLabelToPage = async (
     description?: string
   }
 ): Promise<boolean> => {
-  const user = await getRepository(User).findOne(ctx.uid)
+  const user = await getRepository(User).findOneBy({
+    id: ctx.uid,
+  })
+  if (!user) {
+    return false
+  }
 
-  let labelEntity = await getRepository(Label).findOne({
-    where: {
-      user: user,
-      name: ILike(label.name),
-    },
+  let labelEntity = await getRepository(Label).findOneBy({
+    user: { id: user.id },
+    name: ILike(label.name),
   })
 
   if (!labelEntity) {

packages/api/src/services/newsletters.ts

@@ -1,14 +1,24 @@
-import { getRepository } from 'typeorm'
 import { NewsletterEmail } from '../entity/newsletter_email'
 import { nanoid } from 'nanoid'
 import { User } from '../entity/user'
 import { CreateNewsletterEmailErrorCode } from '../generated/graphql'
 import { env } from '../env'
+import { getRepository } from '../entity/utils'
+import addressparser = require('nodemailer/lib/addressparser')
+
+const parsedAddress = (emailAddress: string): string | undefined => {
+  const res = addressparser(emailAddress, { flatten: true })
+  if (!res || res.length < 1) {
+    return undefined
+  }
+  return res[0].address
+}
 
 export const createNewsletterEmail = async (
   userId: string
 ): Promise<NewsletterEmail> => {
-  const user = await getRepository(User).findOne(userId, {
+  const user = await getRepository(User).findOne({
+    where: { id: userId },
     relations: ['profile'],
   })
   if (!user) {
@@ -19,19 +29,17 @@ export const createNewsletterEmail = async (
   // generate a random email address with username prefix
   const emailAddress = createRandomEmailAddress(user.profile.username, 8)
 
-  return getRepository(NewsletterEmail)
-    .create({
-      address: emailAddress,
-      user: user,
-    })
-    .save()
+  return getRepository(NewsletterEmail).save({
+    address: emailAddress,
+    user: user,
+  })
 }
 
 export const getNewsletterEmails = async (
   userId: string
 ): Promise<NewsletterEmail[]> => {
   return getRepository(NewsletterEmail).find({
-    where: { user: userId },
+    where: { user: { id: userId } },
     order: { createdAt: 'DESC' },
   })
 }
@@ -46,9 +54,10 @@ export const updateConfirmationCode = async (
   emailAddress: string,
   confirmationCode: string
 ): Promise<boolean> => {
+  const address = parsedAddress(emailAddress)
   const result = await getRepository(NewsletterEmail)
     .createQueryBuilder()
-    .where('address ILIKE :address', { address: emailAddress })
+    .where('address ILIKE :address', { address })
     .update({
       confirmationCode: confirmationCode,
     })
@@ -59,11 +68,12 @@ export const updateConfirmationCode = async (
 
 export const getNewsletterEmail = async (
   emailAddress: string
-): Promise<NewsletterEmail | undefined> => {
+): Promise<NewsletterEmail | null> => {
+  const address = parsedAddress(emailAddress)
   return getRepository(NewsletterEmail)
     .createQueryBuilder('newsletter_email')
     .innerJoinAndSelect('newsletter_email.user', 'user')
-    .where('address ILIKE :address', { address: emailAddress })
+    .where('address ILIKE :address', { address })
     .getOne()
 }
 

packages/api/src/services/reports.ts

@@ -1,8 +1,8 @@
-import { getRepository } from 'typeorm'
 import { ReportItemInput, ReportType } from '../generated/graphql'
 import { ContentDisplayReport } from '../entity/reports/content_display_report'
 import { AbuseReport } from '../entity/reports/abuse_report'
 import { getPageById } from '../elastic'
+import { getRepository } from '../entity/utils'
 
 export const saveContentDisplayReport = async (
   uid: string,
@@ -20,16 +20,14 @@ export const saveContentDisplayReport = async (
   // We capture the article content and original html now, in case it
   // reparsed or updated later, this gives us a view of exactly
   // what the user saw.
-  const result = await repo
-    .create({
-      userId: uid,
-      elasticPageId: input.pageId,
-      content: page.content,
-      originalHtml: page.originalHtml || undefined,
-      originalUrl: page.url,
-      reportComment: input.reportComment,
-    })
-    .save()
+  const result = await repo.save({
+    userId: uid,
+    elasticPageId: input.pageId,
+    content: page.content,
+    originalHtml: page.originalHtml || undefined,
+    originalUrl: page.url,
+    reportComment: input.reportComment,
+  })
 
   return !!result
 }
@@ -55,16 +53,14 @@ export const saveAbuseReport = async (
   // We capture the article content and original html now, in case it
   // reparsed or updated later, this gives us a view of exactly
   // what the user saw.
-  const result = await repo
-    .create({
-      reportedBy: uid,
-      sharedBy: input.sharedBy,
-      elasticPageId: input.pageId,
-      itemUrl: input.itemUrl,
-      reportTypes: [ReportType.Abusive],
-      reportComment: input.reportComment,
-    })
-    .save()
+  const result = await repo.save({
+    reportedBy: uid,
+    sharedBy: input.sharedBy,
+    elasticPageId: input.pageId,
+    itemUrl: input.itemUrl,
+    reportTypes: [ReportType.Abusive],
+    reportComment: input.reportComment,
+  })
 
   return !!result
 }

packages/api/src/services/save_email.ts

@@ -12,6 +12,7 @@ import { createPage, getPageByParam, updatePage } from '../elastic'
 export type SaveContext = {
   pubsub: PubsubClient
   uid: string
+  refresh?: boolean
 }
 
 export type SaveEmailInput = {
@@ -67,7 +68,7 @@ export const saveEmail = async (
     readingProgressPercent: 0,
   }
 
-  const page = await getPageByParam({ url: articleToSave.url })
+  const page = await getPageByParam({ userId: ctx.uid, url: articleToSave.url })
   if (page) {
     const result = await updatePage(page.id, { archivedAt: null }, ctx)
     console.log('updated page from email', result)
@@ -82,7 +83,6 @@ export const saveEmail = async (
     return undefined
   }
 
-  console.log('created new page from email', pageId)
   articleToSave.id = pageId
 
   return articleToSave

packages/api/src/services/save_newsletter_email.ts

@@ -22,7 +22,8 @@ interface NewsletterMessage {
 // Returns true if the link was created successfully. Can still fail to
 // send the push but that is ok and we wont retry in that case.
 export const saveNewsletterEmail = async (
-  data: NewsletterMessage
+  data: NewsletterMessage,
+  ctx?: SaveContext
 ): Promise<boolean> => {
   // get user from newsletter email
   const newsletterEmail = await getNewsletterEmail(data.email)
@@ -43,7 +44,7 @@ export const saveNewsletterEmail = async (
     },
   })
 
-  const ctx: SaveContext = {
+  const saveCtx = ctx || {
     pubsub: createPubSubClient(),
     uid: newsletterEmail.user.id,
   }
@@ -55,14 +56,14 @@ export const saveNewsletterEmail = async (
     author: data.author,
   }
 
-  const page = await saveEmail(ctx, input)
+  const page = await saveEmail(saveCtx, input)
   if (!page) {
     console.log('newsletter not created:', input)
     return false
   }
 
   // add newsletters label to page
-  const result = await addLabelToPage(ctx, page.id, {
+  const result = await addLabelToPage(saveCtx, page.id, {
     name: 'Newsletter',
     color: '#07D2D1',
   })

packages/api/src/services/user_device_tokens.ts

@@ -1,34 +1,36 @@
-import { getManager, getRepository } from 'typeorm'
 import { UserDeviceToken } from '../entity/user_device_tokens'
 import { User } from '../entity/user'
 import { SetDeviceTokenErrorCode } from '../generated/graphql'
-import { setClaims } from '../entity/utils'
+import { getRepository, setClaims } from '../entity/utils'
 import { analytics } from '../utils/analytics'
 import { env } from '../env'
+import { AppDataSource } from '../server'
 
 export const getDeviceToken = async (
   id: string
-): Promise<UserDeviceToken | undefined> => {
-  return getRepository(UserDeviceToken).findOne(id)
+): Promise<UserDeviceToken | null> => {
+  return getRepository(UserDeviceToken).findOneBy({ id })
 }
 
 export const getDeviceTokenByToken = async (
   token: string
-): Promise<UserDeviceToken | undefined> => {
-  return getRepository(UserDeviceToken).findOne({ token })
+): Promise<UserDeviceToken | null> => {
+  return getRepository(UserDeviceToken).findOneBy({ token })
 }
 
 export const getDeviceTokensByUserId = async (
   userId: string
 ): Promise<UserDeviceToken[] | undefined> => {
-  return getRepository(UserDeviceToken).find({ where: { user: userId } })
+  return getRepository(UserDeviceToken).find({
+    where: { user: { id: userId } },
+  })
 }
 
 export const createDeviceToken = async (
   userId: string,
   token: string
 ): Promise<UserDeviceToken> => {
-  const user = await getRepository(User).findOne(userId)
+  const user = await getRepository(User).findOneBy({ id: userId })
   if (!user) {
     return Promise.reject({
       errorCode: SetDeviceTokenErrorCode.Unauthorized,
@@ -43,19 +45,17 @@ export const createDeviceToken = async (
     },
   })
 
-  return getRepository(UserDeviceToken)
-    .create({
-      token: token,
-      user: user,
-    })
-    .save()
+  return getRepository(UserDeviceToken).save({
+    token: token,
+    user: user,
+  })
 }
 
 export const deleteDeviceToken = async (
   id: string,
   userId: string
 ): Promise<boolean> => {
-  const user = await getRepository(User).findOne(userId)
+  const user = await getRepository(User).findOneBy({ id: userId })
   if (!user) {
     return Promise.reject({
       errorCode: SetDeviceTokenErrorCode.Unauthorized,
@@ -70,7 +70,7 @@ export const deleteDeviceToken = async (
     },
   })
 
-  return getManager().transaction(async (t) => {
+  return AppDataSource.transaction(async (t) => {
     await setClaims(t, userId)
     const result = await t.getRepository(UserDeviceToken).delete(id)
 

packages/api/test/db.ts

@@ -1,13 +1,5 @@
-import {
-  createConnection,
-  getConnection,
-  getManager,
-  getRepository,
-} from 'typeorm'
-import { SnakeNamingStrategy } from 'typeorm-naming-strategies'
 import Postgrator from 'postgrator'
 import { User } from '../src/entity/user'
-import { createUser } from '../src/services/create_user'
 import { Profile } from '../src/entity/profile'
 import { Page } from '../src/entity/page'
 import { Link } from '../src/entity/link'
@@ -15,6 +7,10 @@ import { Reminder } from '../src/entity/reminder'
 import { NewsletterEmail } from '../src/entity/newsletter_email'
 import { UserDeviceToken } from '../src/entity/user_device_tokens'
 import { Label } from '../src/entity/label'
+import { AppDataSource } from '../src/server'
+import { getRepository } from '../src/entity/utils'
+import { createUser } from '../src/services/create_user'
+import { SnakeNamingStrategy } from 'typeorm-naming-strategies'
 
 const runMigrations = async () => {
   const migrationDirectory = __dirname + '/../../db/migrations'
@@ -43,8 +39,10 @@ const runMigrations = async () => {
   }
 }
 
-const createEntityConnection = async (): Promise<void> => {
-  await createConnection({
+export const createTestConnection = async (): Promise<void> => {
+  await runMigrations()
+
+  AppDataSource.setOptions({
     type: 'postgres',
     host: process.env.PG_HOST,
     port: Number(process.env.PG_PORT),
@@ -57,21 +55,11 @@ const createEntityConnection = async (): Promise<void> => {
     subscribers: [__dirname + '/../src/events/**/*{.js,.ts}'],
     namingStrategy: new SnakeNamingStrategy(),
   })
-}
-
-export const createTestConnection = async (): Promise<void> => {
-  try {
-    getConnection()
-    // eslint-disable-next-line no-empty
-  } catch (error) {}
-
-  await runMigrations()
-  await createEntityConnection()
+  await AppDataSource.initialize()
 }
 
 export const deleteTestUser = async (name: string) => {
-  await getConnection()
-    .createQueryBuilder()
+  await AppDataSource.createQueryBuilder()
     .delete()
     .from(User)
     .where({ email: `${name}@fake.com` })
@@ -93,68 +81,58 @@ export const createTestUser = async (
     inviteCode: invite,
     password: password,
   })
+
   return newUser
 }
 
 export const createUserWithoutProfile = async (name: string): Promise<User> => {
-  return getManager()
-    .getRepository(User)
-    .create({
-      source: 'GOOGLE',
-      sourceUserId: 'fake-user-id-' + name,
-      email: `${name}@fake.com`,
-      name: name,
-    })
-    .save()
+  return getRepository(User).save({
+    source: 'GOOGLE',
+    sourceUserId: 'fake-user-id-' + name,
+    email: `${name}@fake.com`,
+    name: name,
+  })
 }
 
-export const getProfile = async (user: User): Promise<Profile | undefined> => {
-  return Profile.findOne({ where: { user: user } })
+export const getProfile = async (user: User): Promise<Profile | null> => {
+  return getRepository(Profile).findOneBy({ user: { id: user.id } })
 }
 
 export const createTestPage = async (): Promise<Page> => {
-  return getRepository(Page)
-    .create({
-      originalHtml: 'html',
-      content: 'Test content',
-      description: 'Test description',
-      title: 'Test title',
-      author: 'Test author',
-      url: 'Test url',
-      hash: 'Test hash',
-    })
-    .save()
+  return getRepository(Page).save({
+    originalHtml: 'html',
+    content: 'Test content',
+    description: 'Test description',
+    title: 'Test title',
+    author: 'Test author',
+    url: 'Test url',
+    hash: 'Test hash',
+  })
 }
 
 export const createTestLink = async (user: User, page: Page): Promise<Link> => {
-  return getRepository(Link)
-    .create({
-      user: user,
-      page: page,
-      slug: 'Test slug',
-      articleUrl: 'Test url',
-      articleHash: 'Test hash',
-    })
-    .save()
+  return getRepository(Link).save({
+    user: user,
+    page: page,
+    slug: 'Test slug',
+    articleUrl: 'Test url',
+    articleHash: 'Test hash',
+  })
 }
 
 export const createTestReminder = async (
   user: User,
   link?: string
 ): Promise<Reminder> => {
-  return getRepository(Reminder)
-    .create({
-      user: user,
-      link: link,
-      remindAt: new Date(),
-    })
-    .save()
+  return getRepository(Reminder).save({
+    user: user,
+    link: link,
+    remindAt: new Date(),
+  })
 }
 
-export const getReminder = async (
-  id: string
-): Promise<Reminder | undefined> => {
-  return getRepository(Reminder).findOne(id)
+export const getReminder = async (id: string): Promise<Reminder | null> => {
+  return getRepository(Reminder).findOneBy({ id })
 }
 
 export const createTestNewsletterEmail = async (
@@ -162,44 +140,40 @@ export const createTestNewsletterEmail = async (
   emailAddress?: string,
   confirmationCode?: string
 ): Promise<NewsletterEmail> => {
-  return getRepository(NewsletterEmail)
-    .create({
-      user: user,
-      address: emailAddress,
-      confirmationCode: confirmationCode,
-    })
-    .save()
+  return getRepository(NewsletterEmail).save({
+    user: user,
+    address: emailAddress,
+    confirmationCode: confirmationCode,
+  })
 }
 
 export const getNewsletterEmail = async (
   id: string
-): Promise<NewsletterEmail | undefined> => {
-  return getRepository(NewsletterEmail).findOne(id)
+): Promise<NewsletterEmail | null> => {
+  return getRepository(NewsletterEmail).findOneBy({ id })
 }
 
 export const createTestDeviceToken = async (
   user: User
 ): Promise<UserDeviceToken> => {
-  return getRepository(UserDeviceToken)
-    .create({
-      user: user,
-      token: 'Test token',
-    })
-    .save()
+  return getRepository(UserDeviceToken).save({
+    user: user,
+    token: 'Test token',
+  })
 }
 
 export const getDeviceToken = async (
   id: string
-): Promise<UserDeviceToken | undefined> => {
-  return getRepository(UserDeviceToken).findOne(id)
+): Promise<UserDeviceToken | null> => {
+  return getRepository(UserDeviceToken).findOneBy({ id })
 }
 
-export const getUser = async (id: string): Promise<User | undefined> => {
-  return getRepository(User).findOne(id)
+export const getUser = async (id: string): Promise<User | null> => {
+  return getRepository(User).findOneBy({ id })
 }
 
-export const getLink = async (id: string): Promise<Link | undefined> => {
-  return getRepository(Link).findOne(id)
+export const getLink = async (id: string): Promise<Link | null> => {
+  return getRepository(Link).findOneBy({ id })
 }
 
 export const createTestLabel = async (
@@ -207,11 +181,9 @@ export const createTestLabel = async (
   name: string,
   color: string
 ): Promise<Label> => {
-  return getRepository(Label)
-    .create({
-      user: user,
-      name: name,
-      color: color,
-    })
-    .save()
+  return getRepository(Label).save({
+    user: user,
+    name: name,
+    color: color,
+  })
 }

packages/api/test/elastic/index.test.ts

@@ -113,7 +113,6 @@ describe('elastic api', () => {
   describe('getPageById', () => {
     it('gets a page by id', async () => {
       const pageFound = await getPageById(page.id)
-
       expect(pageFound).not.undefined
     })
   })
@@ -128,7 +127,6 @@ describe('elastic api', () => {
       await updatePage(page.id, updatedPageData, ctx)
 
       const updatedPage = await getPageById(page.id)
-
       expect(updatedPage?.title).to.eql(newTitle)
     })
   })

packages/api/test/global-teardown.ts

@@ -1,6 +1,6 @@
-import { getConnection } from 'typeorm'
+import { AppDataSource } from '../src/server'
 
 export const mochaGlobalTeardown = async () => {
-  await getConnection().close()
+  await AppDataSource.destroy()
   console.log('db connection closed')
 }

packages/api/test/gql/sanitize-directive.test.ts

@@ -1,12 +1,5 @@
-import { createTestUser, deleteTestUser, getProfile, getUser } from '../db'
+import { createTestUser, deleteTestUser } from '../db'
 import { graphqlRequest, request } from '../util'
-import { expect } from 'chai'
-import {
-  LoginErrorCode,
-  SignupErrorCode,
-  UpdateUserErrorCode,
-  UpdateUserProfileErrorCode,
-} from '../../src/generated/graphql'
 import { User } from '../../src/entity/user'
 import { hashPassword } from '../../src/utils/auth'
 import 'mocha'
@@ -33,7 +26,7 @@ describe('Sanitize Directive', () => {
   })
 
   describe('Update user with a bio that is too long', () => {
-    let bio = "".padStart(500, '*');
+    let bio = ''.padStart(500, '*')
     let query: string
 
     beforeEach(() => {

packages/api/test/resolvers/article.test.ts

@@ -19,9 +19,9 @@ import {
 } from '../../src/elastic'
 import { PageType, UploadFileStatus } from '../../src/generated/graphql'
 import { Page, PageContext } from '../../src/elastic/types'
-import { getRepository } from 'typeorm'
 import { UploadFile } from '../../src/entity/upload_file'
 import { createPubSubClient } from '../../src/datalayer/pubsub'
+import { getRepository } from '../../src/entity/utils'
 
 chai.use(chaiString)
 

packages/api/test/resolvers/highlight.test.ts

@@ -137,7 +137,7 @@ describe('Highlights API', () => {
     before(async () => {
       // create test highlight
       highlightId = generateFakeUuid()
-      const shortHighlightId = '_short_id'
+      const shortHighlightId = '_short_id_1'
       const query = createHighlightQuery(
         authToken,
         pageId,
@@ -149,7 +149,7 @@ describe('Highlights API', () => {
 
     it('should not fail', async () => {
       const newHighlightId = generateFakeUuid()
-      const newShortHighlightId = '_short_id_1'
+      const newShortHighlightId = '_short_id_2'
       const query = mergeHighlightQuery(
         pageId,
         newHighlightId,

packages/api/test/resolvers/labels.test.ts

@@ -7,11 +7,11 @@ import {
 } from '../util'
 import { Label } from '../../src/entity/label'
 import { expect } from 'chai'
-import { getRepository } from 'typeorm'
 import 'mocha'
 import { User } from '../../src/entity/user'
 import { Page } from '../../src/elastic/types'
 import { getPageById } from '../../src/elastic'
+import { getRepository } from '../../src/entity/utils'
 
 describe('Labels API', () => {
   const username = 'fakeUser'
@@ -76,7 +76,9 @@ describe('Labels API', () => {
     it('should return labels', async () => {
       const res = await graphqlRequest(query, authToken).expect(200)
 
-      const labels = await getRepository(Label).find({ where: { user } })
+      const labels = await getRepository(Label).findBy({
+        user: { id: user.id },
+      })
       expect(res.body.data.labels.labels).to.eql(
         labels.map((label) => ({
           id: label.id,
@@ -137,9 +139,9 @@ describe('Labels API', () => {
 
       it('should create label', async () => {
         const res = await graphqlRequest(query, authToken).expect(200)
-        const label = await getRepository(Label).findOne(
-          res.body.data.createLabel.label.id
-        )
+        const label = await getRepository(Label).findOneBy({
+          id: res.body.data.createLabel.label.id,
+        })
         expect(label).to.exist
       })
     })
@@ -203,7 +205,7 @@ describe('Labels API', () => {
 
       it('should delete label', async () => {
         await graphqlRequest(query, authToken).expect(200)
-        const label = await getRepository(Label).findOne(labelId)
+        const label = await getRepository(Label).findOneBy({ id: labelId })
         expect(label).to.not.exist
       })
     })

packages/api/test/resolvers/newsletters.test.ts

@@ -28,11 +28,11 @@ describe('Newsletters API', () => {
     //  create test newsletter emails
     const newsletterEmail1 = await createTestNewsletterEmail(
       user,
-      'Test_email_address_1'
+      'Test_email_address_1@fake-email.com'
     )
     const newsletterEmail2 = await createTestNewsletterEmail(
       user,
-      'Test_email_address_2'
+      'Test_email_address_2@fake-email.com'
     )
     newsletterEmails = [newsletterEmail1, newsletterEmail2]
   })
@@ -164,7 +164,7 @@ describe('Newsletters API', () => {
         const newsletterEmail = await getNewsletterEmail(
           response.body.data.deleteNewsletterEmail.newsletterEmail.id
         )
-        expect(newsletterEmail).to.be.undefined
+        expect(newsletterEmail).to.be.null
       })
     })
 

packages/api/test/resolvers/report.test.ts

@@ -3,9 +3,9 @@ import { Page } from '../../src/elastic/types'
 import { createTestUser, deleteTestUser } from '../db'
 import { createTestElasticPage, graphqlRequest, request } from '../util'
 import { ReportType } from '../../src/generated/graphql'
-import { getRepository } from 'typeorm'
 import { ContentDisplayReport } from '../../src/entity/reports/content_display_report'
 import { expect } from 'chai'
+import { getRepository } from '../../src/entity/utils'
 
 describe('Report API', () => {
   const username = 'fakeUser'
@@ -70,7 +70,7 @@ describe('Report API', () => {
         await graphqlRequest(query, authToken).expect(200)
 
         expect(
-          await getRepository(ContentDisplayReport).find({
+          await getRepository(ContentDisplayReport).findBy({
             elasticPageId: pageId,
           })
         ).to.exist

packages/api/test/resolvers/user_device_tokens.test.ts

@@ -75,7 +75,7 @@ describe('Device tokens API', () => {
           const deviceToken = await getDeviceToken(
             response.body.data.setDeviceToken.deviceToken.id
           )
-          expect(deviceToken).to.be.undefined
+          expect(deviceToken).to.be.null
         })
       })
 
@@ -105,7 +105,7 @@ describe('Device tokens API', () => {
         const deviceToken = await getDeviceToken(
           response.body.data.setDeviceToken.deviceToken.id
         )
-        expect(deviceToken).not.to.be.undefined
+        expect(deviceToken).not.to.be.null
       })
     })
 

packages/api/test/resolvers/user_feed_article.test.ts

@@ -10,8 +10,8 @@ import { SharedArticleErrorCode } from '../../src/generated/graphql'
 import { Page } from '../../src/entity/page'
 import { Link } from '../../src/entity/link'
 import { Highlight } from '../../src/entity/highlight'
-import { getRepository } from 'typeorm'
 import 'mocha'
+import { getRepository } from '../../src/entity/utils'
 
 describe('User feed article API', () => {
   const existingUsername = 'fakeUser'

packages/api/test/routers/article_router.test.ts

@@ -3,6 +3,7 @@ import { request } from '../util'
 import { expect } from 'chai'
 import nock from 'nock'
 import 'mocha'
+import { env } from '../../src/env'
 
 describe('/article/save API', () => {
   const username = 'fakeUser'
@@ -12,7 +13,7 @@ describe('/article/save API', () => {
   // We need to mock the pupeeteer-parse
   // service here because in dev mode the task gets
   // called immediately.
-  nock('http://localhost:8080/').post('/').reply(200)
+  nock(env.queue.puppeteerTaskHanderUrl).post('/').reply(200)
 
   before(async () => {
     // create test user and login

packages/api/test/routers/pdf_attachments.test.ts

@@ -12,7 +12,7 @@ import { getPageById } from '../../src/elastic'
 
 describe('PDF attachments Router', () => {
   const username = 'fakeUser'
-  const newsletterEmail = 'fakeEmail'
+  const newsletterEmail = 'fakeEmail@fake-email.com'
 
   let user: User
   let authToken: string

packages/api/test/services/labels.test.ts

@@ -1,7 +1,6 @@
 import 'mocha'
 import { expect } from 'chai'
 import 'chai/register-should'
-import { labelsLoader } from '../../src/services/labels'
 import {
   createTestLabel,
   createTestLink,
@@ -9,10 +8,11 @@ import {
   createTestUser,
   deleteTestUser,
 } from '../db'
-import { getRepository } from 'typeorm'
 import { LinkLabel } from '../../src/entity/link_label'
 import { Label } from '../../src/entity/label'
 import { Link } from '../../src/entity/link'
+import { labelsLoader } from '../../src/services/labels'
+import { getRepository } from '../../src/entity/utils'
 
 describe('batch get labels from linkIds', () => {
   let username = 'testUser'

packages/api/test/services/save_email.test.ts

@@ -17,6 +17,7 @@ describe('saveEmail', () => {
     const ctx: SaveContext = {
       pubsub: createPubSubClient(),
       uid: user.id,
+      refresh: true,
     }
 
     await saveEmail(ctx, {
@@ -36,15 +37,11 @@ describe('saveEmail', () => {
     })
     expect(secondResult).to.not.be.undefined
 
-    setTimeout(async () => {
-      const page = await getPageByParam({ userId: user.id })
-      if (!page) {
-        expect.fail('page not found')
-      }
-      expect(page.url).to.equal('https://example.com')
-      expect(page.title).to.equal('fake title')
-      expect(page.author).to.equal('fake author')
-      expect(page.content).to.contain('fake content')
-    })
+    const page = await getPageByParam({ userId: user.id })
+    expect(page).to.exist
+    expect(page?.url).to.equal('https://example.com')
+    expect(page?.title).to.equal('fake title')
+    expect(page?.author).to.equal('fake author')
+    expect(page?.content).to.contain('fake content')
   })
 })

packages/api/test/services/save_newsletter_email.test.ts

@@ -7,16 +7,24 @@ import { saveNewsletterEmail } from '../../src/services/save_newsletter_email'
 import { getPageByParam } from '../../src/elastic'
 import { User } from '../../src/entity/user'
 import { NewsletterEmail } from '../../src/entity/newsletter_email'
+import { SaveContext } from '../../src/services/save_email'
+import { createPubSubClient } from '../../src/datalayer/pubsub'
 
 describe('saveNewsletterEmail', () => {
   const username = 'fakeUser'
 
   let user: User
   let email: NewsletterEmail
+  let ctx: SaveContext
 
   before(async () => {
     user = await createTestUser(username)
     email = await createNewsletterEmail(user.id)
+    ctx = {
+      pubsub: createPubSubClient(),
+      refresh: true,
+      uid: user.id,
+    }
   })
 
   after(async () => {
@@ -30,7 +38,7 @@ describe('saveNewsletterEmail', () => {
       url: 'https://example.com',
       title: 'fake title',
       author: 'fake author',
-    })
+    }, ctx)
 
     setTimeout(async () => {
       const page = await getPageByParam({ userId: user.id })
@@ -56,7 +64,7 @@ describe('saveNewsletterEmail', () => {
       url: 'https://example.com/2',
       title: 'fake title',
       author: 'fake author',
-    })
+    }, ctx)
 
     setTimeout(async () => {
       const page = await getPageByParam({ userId: user.id })

packages/api/test/util.ts

@@ -4,7 +4,7 @@ import { v4 } from 'uuid'
 import { corsConfig } from '../src/utils/corsConfig'
 import { Page } from '../src/elastic/types'
 import { PageType } from '../src/generated/graphql'
-import { createPage } from '../src/elastic'
+import { createPage, getPageById } from '../src/elastic'
 import { User } from '../src/entity/user'
 import { Label } from '../src/entity/label'
 import { createPubSubClient } from '../src/datalayer/pubsub'
@@ -62,5 +62,11 @@ export const createTestElasticPage = async (
   if (pageId) {
     page.id = pageId
   }
-  return page
+
+  const res = await getPageById(page.id)
+  console.log('got page', res)
+  if (!res) {
+    throw new Error('Failed to create page')
+  }
+  return res
 }

packages/api/test/utils/parser.test.ts

@@ -30,12 +30,22 @@ describe('isProbablyNewsletter', () => {
 
 describe('findNewsletterUrl', async () => {
   it('gets the URL from the header if it is a substack newsletter', async () => {
+    nock('https://newsletter.slowchinese.net')
+    .head('/p/companies-that-eat-people-217?token=eyJ1c2VyX2lkIjoxMTU0MzM0NSwicG9zdF9pZCI6NDg3MjA5NDAsImlhdCI6MTY0NTI1NzQ1MSwiaXNzIjoicHViLTI4MDUzMSIsInN1YiI6InBvc3QtcmVhY3Rpb24ifQ.l5F3Kx6K9tvy9cRAXx3MepobQBCJDJQgAxOpA0INIZA')
+    .reply(200, '');
     const html = load('./test/utils/data/substack-forwarded-newsletter.html')
     const url = await findNewsletterUrl(html)
     // Not sure if the redirects from substack expire, this test could eventually fail
     expect(url).to.startWith('https://newsletter.slowchinese.net/p/companies-that-eat-people-217')
   })
   it('gets the URL from the header if it is a beehiiv newsletter', async () => {
+    nock('https://u23463625.ct.sendgrid.net')
+    .head('/ss/c/AX1lEgEQaxtvFxLaVo0GBo_geajNrlI1TGeIcmMViR3pL3fEDZnbbkoeKcaY62QZk0KPFudUiUXc_uMLerV4nA/3k5/3TFZmreTR0qKSCgowABnVg/h30/zzLik7UXd1H_n4oyd5W8Xu639AYQQB2UXz-CsssSnno')
+    .reply(302, undefined,{
+      'Location': 'https://www.milkroad.com/p/talked-guy-spent-30m-beeple'
+    })
+    .get('/p/talked-guy-spent-30m-beeple')
+    .reply(200, '');
     const html = load('./test/utils/data/beehiiv-newsletter.html')
     const url = await findNewsletterUrl(html)
     expect(url).to.startWith('https://www.milkroad.com/p/talked-guy-spent-30m-beeple')

packages/cypress/README.md

@@ -0,0 +1,2 @@
+
+Run `yarn && yarn cypress open`

packages/cypress/cypress.json

@@ -0,0 +1,8 @@
+{
+  "baseUrl": "http://localhost:3000",
+  "experimentalSessionSupport": true,
+    "fixturesFolder": false,
+  "supportFile": "cypress/support/index.js",
+  "pluginsFile": false,
+  "video": false
+}

packages/cypress/cypress/integration/authentication/register-w-email.js

@@ -0,0 +1,34 @@
+const email = 'tester@omnivore.app'
+const username = 'testuser'
+const password = 'testpassword'
+const fullName = 'Test User'
+
+describe('Register with email', () => {
+  it('creates a new user', function () {
+    cy.visit('/email-registration')
+
+    cy.get('input[name=email]').type(email)
+    cy.get('input[name=username]').type(username)
+    cy.get('input[name=password]').type(password)
+    cy.get('input[name=name]').type(fullName)
+
+    cy.get('form').submit()
+
+    // we should be redirected to /dashboard
+    cy.location('pathname').should('include', '/email-login')
+  })
+})
+
+describe('Login with email', () => {
+  it('sets auth token and redirects', function () {
+    cy.visit('/email-login')
+
+    cy.get('input[name=email]').type(email)
+    cy.get('input[name=password]').type(password)
+
+    cy.get('form').submit()
+
+    cy.getCookie('auth').should('exist')
+    cy.location('pathname').should('include', '/home')
+  })
+})

packages/cypress/cypress/integration/library/add-item.js

@@ -0,0 +1,22 @@
+
+describe('add link button', () => {
+  before(() => {
+    const email = 'tester@omnivore.app'
+    const password = 'testpassword'
+
+    cy.login(email, password)
+    cy.visit('/home');
+  });
+
+  it('should add a link', () => {
+    // Use keyboard command to open add link modal
+    cy.get('body').type('a')
+
+    cy.focused().type('https://jacksonh.org/{enter}')
+
+    // wait for the link to be added
+    cy.wait(2000)
+
+    cy.reload()
+  });
+});

packages/cypress/cypress/integration/unauthenticated/redirects.js

@@ -0,0 +1,8 @@
+
+describe('pages that require auth', () => {
+  it('should redirect to login', () => {
+    cy.visit('/home')
+    cy.location('pathname')
+      .should('be.equal', '/login')
+  });
+});

packages/cypress/cypress/plugins/index.js

@@ -0,0 +1,22 @@
+/// <reference types="cypress" />
+// ***********************************************************
+// This example plugins/index.js can be used to load plugins
+//
+// You can change the location of this file or turn off loading
+// the plugins file with the 'pluginsFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/plugins-guide
+// ***********************************************************
+
+// This function is called when a project is opened or re-opened (e.g. due to
+// the project's config changing)
+
+/**
+ * @type {Cypress.PluginConfig}
+ */
+// eslint-disable-next-line no-unused-vars
+module.exports = (on, config) => {
+  // `on` is used to hook into various events Cypress emits
+  // `config` is the resolved Cypress config
+}

packages/cypress/cypress/support/commands.js

@@ -0,0 +1,16 @@
+
+import '@testing-library/cypress/add-commands';
+
+Cypress.Commands.add('login', (email, password) => {
+  cy.session([email, password], () => {
+    cy.visit('/email-login')
+
+    cy.get('input[name=email]').type(email)
+    cy.get('input[name=password]').type(password)
+
+    cy.get('form').submit()
+
+    cy.getCookie('auth').should('exist')
+    cy.location('pathname').should('include', '/home')
+  })
+})

packages/cypress/cypress/support/index.js

@@ -0,0 +1,20 @@
+// ***********************************************************
+// This example support/index.js is processed and
+// loaded automatically before your test files.
+//
+// This is a great place to put global configuration and
+// behavior that modifies Cypress.
+//
+// You can change the location of this file or turn off
+// automatically serving support files with the
+// 'supportFile' configuration option.
+//
+// You can read more here:
+// https://on.cypress.io/configuration
+// ***********************************************************
+
+// Import commands.js using ES2015 syntax:
+import './commands'
+
+// Alternatively you can use CommonJS syntax:
+// require('./commands')

packages/cypress/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@omnivore/cypress",
+  "version": "1.0.0",
+  "private": true,
+  "scripts": {},
+  "dependencies": {
+    "@testing-library/cypress": "^8.0.2",
+    "cypress": "^9.5.3"
+  },
+  "devDependencies": {},
+  "volta": {
+    "node": "14.18.0",
+    "yarn": "1.22.10"
+  }
+}

packages/cypress/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "types": ["node"],
+    "incremental": true
+  },
+  "include": ["**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules"]
+}

packages/db/setup.sh

@@ -4,4 +4,4 @@ psql --host $PG_HOST -U $PG_USER -d $PG_DB -c "CREATE USER app_user WITH PASSWOR
 echo "created app_user"
 yarn workspace @omnivore/db migrate
 psql --host $PG_HOST -U $PG_USER -d $PG_DB -c "GRANT omnivore_user TO app_user;"
-echo "granted omnivore_user to app_user"
+echo "granted omnivore_user to app_user"

packages/inbound-email-handler/src/bloomberg-handler.ts

@@ -5,6 +5,6 @@ export class BloombergHandler extends NewsletterHandler {
     super()
     this.senderRegex = /<.+@mail.bloomberg.*.com>/
     this.urlRegex = /<a class="view-in-browser__url" href=["']([^"']*)["']/
-    this.defaultUrl = 'https://www.bloomberg.com/'
+    this.defaultUrl = 'https://www.bloomberg.com'
   }
 }

packages/inbound-email-handler/src/golang-handler.ts

@@ -5,6 +5,6 @@ export class GolangHandler extends NewsletterHandler {
     super()
     this.senderRegex = /<.+@golangweekly.com>/
     this.urlRegex = /<a href=["']([^"']*)["'].*>Read on the Web<\/a>/
-    this.defaultUrl = 'https://golangweekly.com/'
+    this.defaultUrl = 'https://golangweekly.com'
   }
 }

packages/inbound-email-handler/src/index.ts

@@ -30,10 +30,11 @@ const NEWSLETTER_HANDLERS = [
 
 export const getNewsletterHandler = (
   rawUrl: string,
-  from: string
+  from: string,
+  unSubRawUrl: string
 ): NewsletterHandler | undefined => {
   return NEWSLETTER_HANDLERS.find((h) => {
-    return h.isNewsletter(rawUrl, from)
+    return h.isNewsletter(rawUrl, from, unSubRawUrl)
   })
 }
 
@@ -72,9 +73,12 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         ? forwardedAddress.toString()
         : parsed.to
       const rawUrl = headers['list-post'] ? headers['list-post'].toString() : ''
+      const unSubRawUrl = headers['list-unsubscribe']
+        ? headers['list-unsubscribe'].toString()
+        : ''
 
       // check if it is a forwarding confirmation email or newsletter
-      const newsletterHandler = getNewsletterHandler(rawUrl, from)
+      const newsletterHandler = getNewsletterHandler(rawUrl, from, unSubRawUrl)
       try {
         if (newsletterHandler) {
           console.log('handleNewsletter', from, recipientAddress)

packages/inbound-email-handler/src/newsletter.ts

@@ -1,4 +1,5 @@
 import { PubSub } from '@google-cloud/pubsub'
+import { v4 as uuidv4 } from 'uuid'
 import addressparser from 'addressparser'
 
 const pubsub = new PubSub()
@@ -14,10 +15,10 @@ export class NewsletterHandler {
   protected urlRegex = /NEWSLETTER_URL_REGEX/
   protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
 
-  isNewsletter(_rawUrl: string, from: string): boolean {
+  isNewsletter(rawUrl: string, from: string, unSubRawUrl: string): boolean {
     // Axios newsletter is from <xx@axios.com>
     const re = new RegExp(this.senderRegex)
-    return re.test(from)
+    return re.test(from) && (!!rawUrl || !!unSubRawUrl)
   }
 
   getNewsletterUrl(_rawUrl: string, html: string): string | undefined {
@@ -55,8 +56,11 @@ export class NewsletterHandler {
     }
 
     // fallback to default url if newsletter url does not exist
-    const url = this.getNewsletterUrl(rawUrl, html) || this.defaultUrl
-    const author = this.getAuthor(from)
+    // assign a random uuid to the default url to avoid duplicate url
+    const url =
+      this.getNewsletterUrl(rawUrl, html) ||
+      `${this.defaultUrl}?source=newsletters&id=${uuidv4()}`
+    const author = this.getAuthor(from) || 'Unknown'
 
     const message = {
       email: email,

packages/inbound-email-handler/src/substack-handler.ts

@@ -4,7 +4,7 @@ import addressparser from 'addressparser'
 export class SubstackHandler extends NewsletterHandler {
   constructor() {
     super()
-    this.defaultUrl = 'https://www.substack.com/'
+    this.defaultUrl = 'https://www.substack.com'
   }
 
   getNewsletterUrl(rawUrl: string, _html: string): string | undefined {
@@ -15,7 +15,7 @@ export class SubstackHandler extends NewsletterHandler {
       : undefined
   }
 
-  isNewsletter(rawUrl: string, _from: string): boolean {
+  isNewsletter(rawUrl: string, _from: string, _unSubRawUrl: string): boolean {
     return !!rawUrl
   }
 }

packages/inbound-email-handler/test/newsletter.test.ts

@@ -34,26 +34,36 @@ describe('Newsletter email test', () => {
     it('returns SubstackHandler when email is from SubStack', () => {
       const rawUrl = '<https://hongbo130.substack.com/p/tldr>'
 
-      expect(getNewsletterHandler(rawUrl, '')).to.be.instanceof(SubstackHandler)
+      expect(getNewsletterHandler(rawUrl, '', '')).to.be.instanceof(
+        SubstackHandler
+      )
     })
 
     it('returns AxiosHandler when email is from Axios', () => {
       const from = 'Mike Allen <mike@axios.com>'
+      const unSubRawUrl =
+        '<https://axios.com/unsubscribe?email=mike%40axios.com&code=593781109>'
 
-      expect(getNewsletterHandler('', from)).to.be.instanceof(AxiosHandler)
+      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
+        AxiosHandler
+      )
     })
 
     context('when email is from Bloomberg', () => {
       it('should return BloombergHandler when email is from Bloomberg Business', () => {
         const from = 'From: Bloomberg <noreply@mail.bloombergbusiness.com>'
-        expect(getNewsletterHandler('', from)).to.be.instanceof(
+        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
+
+        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
           BloombergHandler
         )
       })
 
       it('should return BloombergHandler when email is from Bloomberg View', () => {
         const from = 'From: Bloomberg <noreply@mail.bloombergview.com>'
-        expect(getNewsletterHandler('', from)).to.be.instanceof(
+        const unSubRawUrl = '<https://bloomberg.com/unsubscribe>'
+
+        expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
           BloombergHandler
         )
       })
@@ -61,7 +71,11 @@ describe('Newsletter email test', () => {
 
     it('should return GolangHandler when email is from Golang Weekly', () => {
       const from = 'Golang Weekly <peter@golangweekly.com>'
-      expect(getNewsletterHandler('', from)).to.be.instanceof(GolangHandler)
+      const unSubRawUrl = '<https://golangweekly.com/unsubscribe>'
+
+      expect(getNewsletterHandler('', from, unSubRawUrl)).to.be.instanceof(
+        GolangHandler
+      )
     })
   })
 

packages/puppeteer-parse/package.json

@@ -8,11 +8,11 @@
     "@google-cloud/storage": "^5.18.1",
     "@sentry/serverless": "^6.13.3",
     "axios": "^0.26.0",
-    "chrome-aws-lambda": "^7.0.0",
+    "chrome-aws-lambda": "^10.1.0",
     "dotenv": "^8.2.0",
     "jsdom": "^19.0.0",
     "jsonwebtoken": "^8.5.1",
-    "luxon": "^1.26.0",
+    "luxon": "^2.3.1",
     "puppeteer-core": "^7.1.0",
     "winston": "^3.3.3"
   },