mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
Merge pull request #1425 from omnivore-app/feat/ios-loading-perf
Improve how we handle loading the initial items and recently saved items on iOS
This commit is contained in:
Hongbo Wu
GitHub
commit
03b3be54b7
24 changed files with 855 additions and 194 deletions
|
|
@ -9,7 +9,7 @@ public class ShareExtensionViewModel: ObservableObject {
|
|||
@Published public var status: ShareExtensionStatus = .processing
|
||||
@Published public var title: String = ""
|
||||
@Published public var url: String?
|
||||
@Published public var iconURL: String?
|
||||
@Published public var highlightData: HighlightData?
|
||||
@Published public var linkedItem: LinkedItem?
|
||||
@Published public var requestId = UUID().uuidString.lowercased()
|
||||
@Published var debugText: String?
|
||||
|
|
@ -84,30 +84,19 @@ public class ShareExtensionViewModel: ObservableObject {
|
|||
DispatchQueue.main.async {
|
||||
self.status = .saved
|
||||
|
||||
let url = URLComponents(string: payload.url)
|
||||
let hostname = URL(string: payload.url)?.host ?? ""
|
||||
|
||||
switch payload.contentType {
|
||||
case let .html(html: _, title: title, iconURL: iconURL):
|
||||
case let .html(html: _, title: title, highlightData: highlightData):
|
||||
self.title = title ?? ""
|
||||
self.iconURL = iconURL
|
||||
self.url = hostname
|
||||
self.highlightData = highlightData
|
||||
case .none:
|
||||
self.url = hostname
|
||||
self.title = payload.url
|
||||
if var url = url {
|
||||
url.path = "/favicon.ico"
|
||||
self.iconURL = url.url?.absoluteString
|
||||
}
|
||||
case let .pdf(localUrl: localUrl):
|
||||
self.url = hostname
|
||||
self.title = PDFUtils.titleFromPdfFile(localUrl.absoluteString)
|
||||
Task {
|
||||
let localThumbnail = try await PDFUtils.createThumbnailFor(inputUrl: localUrl)
|
||||
DispatchQueue.main.async {
|
||||
self.iconURL = localThumbnail?.absoluteString
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -172,6 +161,15 @@ public class ShareExtensionViewModel: ObservableObject {
|
|||
}
|
||||
|
||||
updateStatusOnMain(requestId: newRequestID, newStatus: .synced)
|
||||
|
||||
// Prefetch the newly saved content
|
||||
if let itemID = newRequestID,
|
||||
let currentViewer = services.dataService.currentViewer?.username,
|
||||
(try? await services.dataService.loadArticleContentWithRetries(itemID: itemID, username: currentViewer)) != nil
|
||||
{
|
||||
updateStatusOnMain(requestId: requestId, newStatus: .synced, objectID: linkedItemObjectID)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
|
|
@ -184,12 +182,20 @@ public class ShareExtensionViewModel: ObservableObject {
|
|||
|
||||
if let objectID = objectID {
|
||||
self.linkedItem = self.services.dataService.viewContext.object(with: objectID) as? LinkedItem
|
||||
if let title = self.linkedItem?.title {
|
||||
self.title = title
|
||||
}
|
||||
self.url = self.linkedItem?.pageURLString
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum ShareExtensionStatus {
|
||||
public enum ShareExtensionStatus: Equatable {
|
||||
public static func == (lhs: ShareExtensionStatus, rhs: ShareExtensionStatus) -> Bool {
|
||||
lhs.displayMessage == rhs.displayMessage
|
||||
}
|
||||
|
||||
case processing
|
||||
case saved
|
||||
case synced
|
||||
|
|
|
|||
|
|
@ -12,15 +12,22 @@ public struct ShareExtensionView: View {
|
|||
|
||||
@State var reminderTime: ReminderTime?
|
||||
@State var hideUntilReminded = false
|
||||
@State var editingTitle = false
|
||||
@State var editingLabels = false
|
||||
@State var previousLabels: [LinkedItemLabel]?
|
||||
@State var messageText: String?
|
||||
|
||||
@State var viewState = ViewState.mainView
|
||||
|
||||
enum FocusField: Hashable {
|
||||
case titleEditor
|
||||
}
|
||||
|
||||
enum ViewState {
|
||||
case mainView
|
||||
case editingTitle
|
||||
case editingLabels
|
||||
case viewingHighlight
|
||||
}
|
||||
|
||||
@FocusState private var focusedField: FocusField?
|
||||
|
||||
private func handleReminderTimeSelection(_ selectedTime: ReminderTime) {
|
||||
|
|
@ -44,27 +51,14 @@ public struct ShareExtensionView: View {
|
|||
}
|
||||
}
|
||||
|
||||
private var cloudIconName: String {
|
||||
private var titleColor: Color {
|
||||
switch viewModel.status {
|
||||
case .synced:
|
||||
return "checkmark.icloud"
|
||||
case .saved, .processing:
|
||||
return "icloud"
|
||||
case .failed(error: _), .syncFailed(error: _):
|
||||
return "exclamationmark.icloud"
|
||||
}
|
||||
}
|
||||
|
||||
private var cloudIconColor: Color {
|
||||
switch viewModel.status {
|
||||
case .saved:
|
||||
return .appGrayText
|
||||
case .processing:
|
||||
return .clear
|
||||
case .failed(error: _), .syncFailed(error: _):
|
||||
return .red
|
||||
case .synced:
|
||||
return .blue
|
||||
return .appGreenSuccess
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -81,71 +75,6 @@ public struct ShareExtensionView: View {
|
|||
return nil
|
||||
}
|
||||
|
||||
public var previewCard: some View {
|
||||
HStack {
|
||||
if let iconURLStr = viewModel.iconURL, let iconURL = URL(string: iconURLStr) {
|
||||
if !iconURL.isFileURL {
|
||||
AsyncImage(
|
||||
url: iconURL,
|
||||
content: { image in
|
||||
image
|
||||
.resizable()
|
||||
.aspectRatio(contentMode: .fill)
|
||||
.frame(width: 61, height: 61)
|
||||
.clipped()
|
||||
},
|
||||
placeholder: {
|
||||
Color.appButtonBackground
|
||||
.aspectRatio(contentMode: .fill)
|
||||
.frame(width: 61, height: 61)
|
||||
}
|
||||
)
|
||||
} else {
|
||||
if let localImage = localImage(from: iconURL) {
|
||||
localImage
|
||||
.resizable()
|
||||
.aspectRatio(contentMode: .fill)
|
||||
.frame(width: 61, height: 61)
|
||||
.clipped()
|
||||
} else {
|
||||
Color.appButtonBackground
|
||||
.aspectRatio(contentMode: .fill)
|
||||
.frame(width: 61, height: 61)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Color.appButtonBackground
|
||||
.aspectRatio(contentMode: .fill)
|
||||
.frame(width: 61, height: 61)
|
||||
}
|
||||
|
||||
VStack(alignment: .leading) {
|
||||
Text(viewModel.title ?? "")
|
||||
.lineLimit(1)
|
||||
.foregroundColor(.appGrayTextContrast)
|
||||
.font(Font.system(size: 15, weight: .semibold))
|
||||
Text(viewModel.url ?? "")
|
||||
.lineLimit(1)
|
||||
.foregroundColor(.appGrayText)
|
||||
.font(Font.system(size: 12, weight: .regular))
|
||||
}
|
||||
Spacer()
|
||||
VStack {
|
||||
Spacer()
|
||||
Image(systemName: cloudIconName)
|
||||
.resizable()
|
||||
.aspectRatio(contentMode: .fill)
|
||||
.frame(width: 12, height: 12, alignment: .trailing)
|
||||
.foregroundColor(cloudIconColor)
|
||||
// .padding(.trailing, 6)
|
||||
.padding(EdgeInsets(top: 0, leading: 0, bottom: 8, trailing: 8))
|
||||
}
|
||||
}
|
||||
.background(Color.appButtonBackground)
|
||||
.frame(maxWidth: .infinity, maxHeight: 61)
|
||||
.cornerRadius(8)
|
||||
}
|
||||
|
||||
var isSynced: Bool {
|
||||
switch viewModel.status {
|
||||
case .synced:
|
||||
|
|
@ -166,7 +95,7 @@ public struct ShareExtensionView: View {
|
|||
|
||||
Text(messageText ?? titleText)
|
||||
.font(.appSubheadline)
|
||||
.foregroundColor(isSynced ? .appGreenSuccess : .appGrayText)
|
||||
.foregroundColor(titleColor)
|
||||
|
||||
Spacer()
|
||||
}
|
||||
|
|
@ -179,14 +108,14 @@ public struct ShareExtensionView: View {
|
|||
.font(.appFootnote)
|
||||
.padding(.trailing, 8)
|
||||
.onTapGesture {
|
||||
editingTitle = true
|
||||
viewState = .editingTitle
|
||||
}
|
||||
})
|
||||
.disabled(editingTitle)
|
||||
.opacity(editingTitle ? 0.0 : 1.0)
|
||||
.disabled(viewState == .editingTitle)
|
||||
.opacity(viewState == .editingTitle ? 0.0 : 1.0)
|
||||
|
||||
VStack(alignment: .leading) {
|
||||
if !editingTitle {
|
||||
if viewState != .editingTitle {
|
||||
Text(self.viewModel.title)
|
||||
.font(.appSubheadline)
|
||||
.foregroundColor(.appGrayTextContrast)
|
||||
|
|
@ -211,7 +140,7 @@ public struct ShareExtensionView: View {
|
|||
|
||||
var labelsSection: some View {
|
||||
HStack {
|
||||
if !editingLabels {
|
||||
if viewState != .editingLabels {
|
||||
ZStack {
|
||||
Circle()
|
||||
.foregroundColor(Color.blue)
|
||||
|
|
@ -243,16 +172,76 @@ public struct ShareExtensionView: View {
|
|||
Image(systemName: "chevron.right")
|
||||
.font(.appCallout)
|
||||
} else {
|
||||
ScrollView {
|
||||
LabelsMasonaryView(labels: labelsViewModel.labels,
|
||||
selectedLabels: labelsViewModel.selectedLabels,
|
||||
onLabelTap: onLabelTap)
|
||||
}.background(Color.appButtonBackground)
|
||||
.cornerRadius(8)
|
||||
VStack {
|
||||
ScrollView {
|
||||
LabelsMasonaryView(labels: labelsViewModel.labels,
|
||||
selectedLabels: labelsViewModel.selectedLabels,
|
||||
onLabelTap: onLabelTap)
|
||||
}.background(Color.appButtonBackground)
|
||||
.cornerRadius(8)
|
||||
|
||||
Button(
|
||||
action: { labelsViewModel.showCreateLabelModal = true },
|
||||
label: {
|
||||
HStack {
|
||||
Spacer()
|
||||
Image(systemName: "plus")
|
||||
Text("Create label")
|
||||
Spacer()
|
||||
}
|
||||
}
|
||||
).buttonStyle(RoundedRectButtonStyle(color: .blue, textColor: .white))
|
||||
}
|
||||
}
|
||||
}
|
||||
.padding(16)
|
||||
.frame(maxWidth: .infinity, maxHeight: self.editingLabels ? .infinity : 60)
|
||||
.frame(maxWidth: .infinity, maxHeight: viewState == .editingLabels ? .infinity : 60)
|
||||
.background(Color.appButtonBackground)
|
||||
.cornerRadius(8)
|
||||
}
|
||||
|
||||
var highlightSection: some View {
|
||||
HStack {
|
||||
if viewState != .viewingHighlight {
|
||||
ZStack {
|
||||
Circle()
|
||||
.foregroundColor(Color.appBackground)
|
||||
.frame(width: 34, height: 34)
|
||||
|
||||
Image(systemName: "highlighter")
|
||||
.font(.appCallout)
|
||||
.frame(width: 34, height: 34)
|
||||
.foregroundColor(Color.black)
|
||||
}
|
||||
.padding(.trailing, 8)
|
||||
|
||||
VStack {
|
||||
Text("Highlight")
|
||||
.font(.appSubheadline)
|
||||
.foregroundColor(Color.appGrayTextContrast)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
|
||||
Text(viewModel.highlightData != nil ?
|
||||
viewModel.highlightData!.highlightText
|
||||
: "Select text before saving to create highlight")
|
||||
.font(.appFootnote)
|
||||
.foregroundColor(Color.appGrayText)
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
}
|
||||
|
||||
Spacer()
|
||||
|
||||
Image(systemName: "chevron.right")
|
||||
.font(.appCallout)
|
||||
} else if let highlightText = self.viewModel.highlightData?.highlightText {
|
||||
Text(highlightText)
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity, alignment: .topLeading)
|
||||
.cornerRadius(8)
|
||||
.padding(0)
|
||||
}
|
||||
}
|
||||
.padding(16)
|
||||
.frame(maxWidth: .infinity, maxHeight: viewState == .viewingHighlight ? .infinity : 60)
|
||||
.background(Color.appButtonBackground)
|
||||
.cornerRadius(8)
|
||||
}
|
||||
|
|
@ -351,6 +340,19 @@ public struct ShareExtensionView: View {
|
|||
}
|
||||
}
|
||||
|
||||
var editingViewTitle: String {
|
||||
switch viewState {
|
||||
case .editingTitle:
|
||||
return "Edit Title"
|
||||
case .editingLabels:
|
||||
return "Labels"
|
||||
case .viewingHighlight:
|
||||
return "Highlight"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
public var body: some View {
|
||||
VStack(alignment: .center) {
|
||||
Capsule()
|
||||
|
|
@ -358,7 +360,7 @@ public struct ShareExtensionView: View {
|
|||
.frame(width: 60, height: 4)
|
||||
.padding(.top, 10)
|
||||
|
||||
if !editingLabels, !editingTitle {
|
||||
if viewState == .mainView {
|
||||
titleBar
|
||||
.padding(.top, 10)
|
||||
.padding(.bottom, 12)
|
||||
|
|
@ -366,28 +368,28 @@ public struct ShareExtensionView: View {
|
|||
ZStack {
|
||||
Button(action: {
|
||||
withAnimation {
|
||||
if editingLabels {
|
||||
if viewState == .editingLabels {
|
||||
if let linkedItem = self.viewModel.linkedItem {
|
||||
self.labelsViewModel.selectedLabels = previousLabels ?? []
|
||||
self.labelsViewModel.saveItemLabelChanges(itemID: linkedItem.unwrappedID,
|
||||
dataService: self.viewModel.services.dataService)
|
||||
}
|
||||
}
|
||||
editingTitle = false
|
||||
editingLabels = false
|
||||
viewState = .mainView
|
||||
}
|
||||
}, label: { Text("Cancel") })
|
||||
.frame(maxWidth: .infinity, alignment: .leading)
|
||||
.opacity(viewState == .viewingHighlight ? 0.0 : 1.0)
|
||||
// Don't show viewState when viewing the highlight
|
||||
|
||||
Text(editingTitle ? "Edit Title" : "Labels").bold()
|
||||
Text(editingViewTitle).bold()
|
||||
.frame(maxWidth: .infinity, alignment: .center)
|
||||
|
||||
Button(action: {
|
||||
withAnimation {
|
||||
editingTitle = false
|
||||
editingLabels = false
|
||||
viewState = .mainView
|
||||
|
||||
if editingTitle {
|
||||
if viewState == .editingTitle {
|
||||
if let linkedItem = self.viewModel.linkedItem {
|
||||
viewModel.submitTitleEdit(dataService: self.viewModel.services.dataService,
|
||||
itemID: linkedItem.unwrappedID,
|
||||
|
|
@ -403,11 +405,11 @@ public struct ShareExtensionView: View {
|
|||
.padding(.bottom, 4)
|
||||
}
|
||||
|
||||
if !editingLabels, !editingTitle {
|
||||
if viewState == .mainView {
|
||||
titleBox
|
||||
}
|
||||
|
||||
if editingTitle {
|
||||
if viewState == .editingTitle {
|
||||
ScrollView(showsIndicators: false) {
|
||||
VStack(alignment: .center, spacing: 16) {
|
||||
VStack(alignment: .leading, spacing: 6) {
|
||||
|
|
@ -436,20 +438,29 @@ public struct ShareExtensionView: View {
|
|||
Spacer()
|
||||
}
|
||||
|
||||
if !editingTitle {
|
||||
labelsSection
|
||||
.padding(.top, 12)
|
||||
.onTapGesture {
|
||||
withAnimation {
|
||||
previousLabels = self.labelsViewModel.selectedLabels
|
||||
editingLabels = true
|
||||
if viewState != .editingTitle {
|
||||
if viewState != .viewingHighlight {
|
||||
labelsSection
|
||||
.onTapGesture {
|
||||
withAnimation {
|
||||
previousLabels = self.labelsViewModel.selectedLabels
|
||||
viewState = .editingLabels
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if viewState != .editingLabels {
|
||||
highlightSection
|
||||
.onTapGesture {
|
||||
withAnimation {
|
||||
viewState = .viewingHighlight
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Spacer()
|
||||
|
||||
if !editingLabels, !editingTitle {
|
||||
if viewState == .mainView {
|
||||
Divider()
|
||||
.padding(.bottom, 20)
|
||||
|
||||
|
|
@ -467,6 +478,9 @@ public struct ShareExtensionView: View {
|
|||
.onAppear {
|
||||
viewModel.savePage(extensionContext: extensionContext)
|
||||
}
|
||||
.sheet(isPresented: $labelsViewModel.showCreateLabelModal) {
|
||||
CreateLabelView(viewModel: labelsViewModel)
|
||||
}
|
||||
.environmentObject(viewModel.services.dataService)
|
||||
.task {
|
||||
await labelsViewModel.loadLabelsFromStore(dataService: viewModel.services.dataService)
|
||||
|
|
|
|||
|
|
@ -36,10 +36,6 @@ import Views
|
|||
|
||||
@AppStorage(UserDefaultKey.lastSelectedLinkedItemFilter.rawValue) var appliedFilter = LinkedItemFilter.inbox.rawValue
|
||||
|
||||
@AppStorage(UserDefaultKey.lastItemSyncTime.rawValue) var lastItemSyncTime = DateFormatter.formatterISO8601.string(
|
||||
from: Date(timeIntervalSinceReferenceDate: 0)
|
||||
)
|
||||
|
||||
func handleReaderItemNotification(objectID: NSManagedObjectID, dataService: DataService) {
|
||||
// Pop the current selected item if needed
|
||||
if selectedItem != nil, selectedItem?.objectID != objectID {
|
||||
|
|
@ -91,32 +87,47 @@ import Views
|
|||
items.insert(item, at: 0)
|
||||
}
|
||||
|
||||
func loadItems(dataService: DataService, audioController: AudioController, isRefresh: Bool) async {
|
||||
let syncStartTime = Date()
|
||||
let thisSearchIdx = searchIdx
|
||||
searchIdx += 1
|
||||
|
||||
isLoading = true
|
||||
showLoadingBar = true
|
||||
|
||||
func loadCurrentViewer(dataService: DataService) async {
|
||||
// Cache the viewer
|
||||
if dataService.currentViewer == nil {
|
||||
Task { _ = try? await dataService.fetchViewer() }
|
||||
_ = try? await dataService.fetchViewer()
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch labels if none are available locally
|
||||
func loadLabels(dataService: DataService) async {
|
||||
let fetchRequest: NSFetchRequest<Models.LinkedItemLabel> = LinkedItemLabel.fetchRequest()
|
||||
fetchRequest.fetchLimit = 1
|
||||
|
||||
if (try? dataService.viewContext.count(for: fetchRequest)) == 0 {
|
||||
_ = try? await dataService.labels()
|
||||
}
|
||||
}
|
||||
|
||||
// Sync items if necessary
|
||||
let lastSyncDate = dateFormatter.date(from: lastItemSyncTime) ?? Date(timeIntervalSinceReferenceDate: 0)
|
||||
func syncItems(dataService: DataService, syncStartTime: Date) async {
|
||||
let lastSyncDate = dateFormatter.date(from: dataService.lastItemSyncTime) ?? Date(timeIntervalSinceReferenceDate: 0)
|
||||
let syncResult = try? await dataService.syncLinkedItems(since: lastSyncDate, cursor: nil)
|
||||
|
||||
if syncResult != nil {
|
||||
lastItemSyncTime = dateFormatter.string(from: syncStartTime)
|
||||
dataService.lastItemSyncTime = dateFormatter.string(from: syncStartTime)
|
||||
}
|
||||
|
||||
// If possible start prefetching new pages in the background
|
||||
if let itemIDs = syncResult?.updatedItemIDs,
|
||||
let username = dataService.currentViewer?.username,
|
||||
itemIDs.count > 0
|
||||
{
|
||||
Task.detached(priority: .background) {
|
||||
await dataService.prefetchPages(itemIDs: itemIDs, username: username)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func loadSearchQuery(dataService: DataService, isRefresh: Bool) async {
|
||||
let thisSearchIdx = searchIdx
|
||||
searchIdx += 1
|
||||
|
||||
if thisSearchIdx > 0, thisSearchIdx <= receivedIdx {
|
||||
return
|
||||
}
|
||||
|
||||
let queryResult = try? await dataService.loadLinkedItems(
|
||||
|
|
@ -125,15 +136,6 @@ import Views
|
|||
cursor: isRefresh ? nil : cursor
|
||||
)
|
||||
|
||||
// Search results aren't guaranteed to return in order so this
|
||||
// will discard old results that are returned while a user is typing.
|
||||
// For example if a user types 'Canucks', often the search results
|
||||
// for 'C' are returned after 'Canucks' because it takes the backend
|
||||
// much longer to compute.
|
||||
if thisSearchIdx > 0, thisSearchIdx <= receivedIdx {
|
||||
return
|
||||
}
|
||||
|
||||
if let queryResult = queryResult {
|
||||
let newItems: [LinkedItem] = {
|
||||
var itemObjects = [LinkedItem]()
|
||||
|
|
@ -159,17 +161,34 @@ import Views
|
|||
cursor = queryResult.cursor
|
||||
if let username = dataService.currentViewer?.username {
|
||||
await dataService.prefetchPages(itemIDs: newItems.map(\.unwrappedID), username: username)
|
||||
// Only preload the first item in the list. We are doing this during the beta
|
||||
// because it will kick off the user's future items being automatically transcribed.
|
||||
// This happens because when an article is saved, we check if the user has a recent
|
||||
// listen. If they do, we will automatically transcribe their message.
|
||||
if let first = newItems.filter({ !$0.isPDF }).first?.id {
|
||||
_ = await audioController.preload(itemIDs: [first])
|
||||
}
|
||||
}
|
||||
} else {
|
||||
updateFetchController(dataService: dataService)
|
||||
}
|
||||
}
|
||||
|
||||
func loadItems(dataService: DataService, audioController _: AudioController, isRefresh: Bool) async {
|
||||
let syncStartTime = Date()
|
||||
let start = CFAbsoluteTimeGetCurrent()
|
||||
|
||||
isLoading = true
|
||||
showLoadingBar = true
|
||||
|
||||
await withTaskGroup(of: Void.self) { group in
|
||||
group.addTask { await self.loadCurrentViewer(dataService: dataService) }
|
||||
group.addTask { await self.loadLabels(dataService: dataService) }
|
||||
group.addTask { await self.syncItems(dataService: dataService, syncStartTime: syncStartTime) }
|
||||
await group.waitForAll()
|
||||
}
|
||||
|
||||
if searchTerm.replacingOccurrences(of: " ", with: "").isEmpty {
|
||||
updateFetchController(dataService: dataService)
|
||||
if appliedFilter != LinkedItemFilter.inbox.rawValue {
|
||||
await loadSearchQuery(dataService: dataService, isRefresh: isRefresh)
|
||||
}
|
||||
} else {
|
||||
await loadSearchQuery(dataService: dataService, isRefresh: isRefresh)
|
||||
}
|
||||
|
||||
isLoading = false
|
||||
showLoadingBar = false
|
||||
|
|
|
|||
|
|
@ -77,10 +77,6 @@ struct LabelsMasonaryView: View {
|
|||
}
|
||||
|
||||
private func item(for item: (label: LinkedItemLabel, selected: Bool)) -> some View {
|
||||
if item.selected {
|
||||
print(" -- SELECTED LABEL", item.label.name)
|
||||
}
|
||||
print("GETTING ITERATION", iteration)
|
||||
let chip = TextChip(feedItemLabel: item.label, negated: false, checked: item.selected) { chip in
|
||||
onLabelTap(item.label, chip)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,101 @@
|
|||
import Models
|
||||
import SwiftUI
|
||||
import Utils
|
||||
import Views
|
||||
import WebKit
|
||||
|
||||
struct HighlightViewer: PlatformViewRepresentable {
|
||||
let highlightData: HighlightData
|
||||
|
||||
func makeCoordinator() -> WebReaderCoordinator {
|
||||
WebReaderCoordinator()
|
||||
}
|
||||
|
||||
private func makePlatformView(context: Context) -> WKWebView {
|
||||
let webView = WebViewManager.shared()
|
||||
let contentController = WKUserContentController()
|
||||
|
||||
webView.navigationDelegate = context.coordinator
|
||||
webView.configuration.userContentController = contentController
|
||||
webView.configuration.userContentController.removeAllScriptMessageHandlers()
|
||||
|
||||
#if os(iOS)
|
||||
webView.isOpaque = false
|
||||
webView.backgroundColor = .clear
|
||||
webView.scrollView.delegate = context.coordinator
|
||||
webView.scrollView.contentInset.top = readerViewNavBarHeight
|
||||
webView.scrollView.verticalScrollIndicatorInsets.top = readerViewNavBarHeight
|
||||
webView.configuration.userContentController.add(webView, name: "viewerAction")
|
||||
#else
|
||||
webView.setValue(false, forKey: "drawsBackground")
|
||||
#endif
|
||||
|
||||
for action in WebViewAction.allCases {
|
||||
webView.configuration.userContentController.add(context.coordinator, name: action.rawValue)
|
||||
}
|
||||
|
||||
webView.configuration.userContentController.addScriptMessageHandler(
|
||||
context.coordinator, contentWorld: .page, name: "articleAction"
|
||||
)
|
||||
|
||||
loadContent(webView: webView)
|
||||
|
||||
return webView
|
||||
}
|
||||
|
||||
private func updatePlatformView(_: WKWebView, context _: Context) {
|
||||
// If the webview had been terminated `needsReload` will have been set to true
|
||||
// Or if the articleContent value has changed then it's id will be different from the coordinator's
|
||||
// if context.coordinator.needsReload {
|
||||
// loadContent(webView: webView)
|
||||
// context.coordinator.needsReload = false
|
||||
// return
|
||||
// }
|
||||
}
|
||||
|
||||
private func loadContent(webView: WKWebView) {
|
||||
let themeKey = ThemeManager.currentThemeName
|
||||
let content = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0, user-scalable=no' />
|
||||
<style>
|
||||
@import url("highlight\(themeKey == "Gray" ? "-dark" : "").css");
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root" />
|
||||
<div id='_omnivore-highlight' class="highlight">
|
||||
\(highlightData.highlightHTML)
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
webView.loadHTMLString(content, baseURL: ViewsPackage.resourceURL)
|
||||
}
|
||||
}
|
||||
|
||||
#if os(iOS)
|
||||
extension HighlightViewer {
|
||||
func makeUIView(context: Context) -> WKWebView {
|
||||
makePlatformView(context: context)
|
||||
}
|
||||
|
||||
func updateUIView(_ webView: WKWebView, context: Context) {
|
||||
updatePlatformView(webView, context: context)
|
||||
}
|
||||
}
|
||||
#else
|
||||
extension WebReader {
|
||||
func makeNSView(context: Context) -> WKWebView {
|
||||
makePlatformView(context: context)
|
||||
}
|
||||
|
||||
func updateNSView(_ webView: WKWebView, context: Context) {
|
||||
updatePlatformView(webView, context: context)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
|
@ -257,8 +257,9 @@ struct WebReaderContainerView: View {
|
|||
.frame(height: readerViewNavBarHeight * navBarVisibilityRatio)
|
||||
.opacity(navBarVisibilityRatio)
|
||||
.background(Color.systemBackground)
|
||||
.alert("Are you sure?", isPresented: $showDeleteConfirmation) {
|
||||
Button("Remove Link", role: .destructive) {
|
||||
.alert("Are you sure you want to remove this item? All associated notes and highlights will be deleted.",
|
||||
isPresented: $showDeleteConfirmation) {
|
||||
Button("Remove Item", role: .destructive) {
|
||||
Snackbar.show(message: "Link removed")
|
||||
dataService.removeLink(objectID: item.objectID)
|
||||
#if os(iOS)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,16 @@ public struct LinkedItemQueryResult {
|
|||
}
|
||||
}
|
||||
|
||||
public struct LinkedItemSyncResult {
|
||||
public let updatedItemIDs: [String]
|
||||
public let cursor: String?
|
||||
|
||||
public init(updatedItemIDs: [String], cursor: String?) {
|
||||
self.updatedItemIDs = updatedItemIDs
|
||||
self.cursor = cursor
|
||||
}
|
||||
}
|
||||
|
||||
public struct LinkedItemAudioProperties {
|
||||
public let itemID: String
|
||||
public let objectID: NSManagedObjectID
|
||||
|
|
|
|||
|
|
@ -8,11 +8,26 @@ import UniformTypeIdentifiers
|
|||
|
||||
let URLREGEX = #"[(http(s)?):\/\/(www\.)?a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)"#
|
||||
|
||||
public struct HighlightData {
|
||||
public let highlightHTML: String
|
||||
public let highlightText: String
|
||||
|
||||
public static func make(dict: NSDictionary?) -> HighlightData? {
|
||||
if let dict = dict,
|
||||
let highlightHTML = dict["highlightHTML"] as? String,
|
||||
let highlightText = dict["highlightText"] as? String
|
||||
{
|
||||
return HighlightData(highlightHTML: highlightHTML, highlightText: highlightText)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
public struct PageScrapePayload {
|
||||
public enum ContentType {
|
||||
case none
|
||||
case html(html: String, title: String?, iconURL: String?)
|
||||
case pdf(localUrl: URL)
|
||||
case html(html: String, title: String?, highlightData: HighlightData?)
|
||||
}
|
||||
|
||||
public let url: String
|
||||
|
|
@ -33,9 +48,9 @@ public struct PageScrapePayload {
|
|||
self.contentType = .pdf(localUrl: localUrl)
|
||||
}
|
||||
|
||||
init(url: String, title: String?, html: String, iconURL: String? = nil) {
|
||||
init(url: String, title: String?, html: String, highlightData: HighlightData?) {
|
||||
self.url = url
|
||||
self.contentType = .html(html: html, title: title, iconURL: iconURL)
|
||||
self.contentType = .html(html: html, title: title, highlightData: highlightData)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -302,7 +317,6 @@ private extension PageScrapePayload {
|
|||
guard let url = results?["url"] as? String else { return nil }
|
||||
let html = results?["originalHTML"] as? String
|
||||
let title = results?["title"] as? String
|
||||
let iconURL = results?["iconURL"] as? String
|
||||
let contentType = results?["contentType"] as? String
|
||||
|
||||
// If we were not able to capture any HTML, treat this as a URL and
|
||||
|
|
@ -318,7 +332,10 @@ private extension PageScrapePayload {
|
|||
}
|
||||
|
||||
if let html = html {
|
||||
return PageScrapePayload(url: url, title: title, html: html, iconURL: iconURL)
|
||||
return PageScrapePayload(url: url,
|
||||
title: title,
|
||||
html: html,
|
||||
highlightData: HighlightData.make(dict: results))
|
||||
}
|
||||
|
||||
return PageScrapePayload(url: url)
|
||||
|
|
|
|||
|
|
@ -110,7 +110,15 @@ extension DataService {
|
|||
let highlightObjects = articleProps.highlights.map {
|
||||
$0.asManagedObject(context: self.backgroundContext)
|
||||
}
|
||||
linkedItem.addToHighlights(NSSet(array: highlightObjects))
|
||||
|
||||
let unsyncedHighlights = existingItem?.highlights?.filter { highlight in
|
||||
if let highlight = highlight as? Highlight, highlight.serverSyncStatus == ServerSyncStatus.isNSync.rawValue {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}.compactMap { $0 as? Highlight } ?? []
|
||||
|
||||
linkedItem.highlights = NSSet(array: highlightObjects + unsyncedHighlights)
|
||||
linkedItem.htmlContent = articleProps.htmlContent
|
||||
linkedItem.id = articleProps.item.id
|
||||
linkedItem.state = articleProps.item.state.rawValue
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import Foundation
|
|||
import Models
|
||||
import OSLog
|
||||
import QuickLookThumbnailing
|
||||
import SwiftUI
|
||||
import Utils
|
||||
|
||||
#if os(iOS)
|
||||
|
|
@ -28,6 +29,10 @@ public final class DataService: ObservableObject {
|
|||
persistentContainer.viewContext
|
||||
}
|
||||
|
||||
@AppStorage(UserDefaultKey.lastItemSyncTime.rawValue) public var lastItemSyncTime = DateFormatter.formatterISO8601.string(
|
||||
from: Date(timeIntervalSinceReferenceDate: 0)
|
||||
)
|
||||
|
||||
public init(appEnvironment: AppEnvironment, networker: Networker) {
|
||||
self.appEnvironment = appEnvironment
|
||||
self.networker = networker
|
||||
|
|
@ -96,10 +101,7 @@ public final class DataService: ObservableObject {
|
|||
}
|
||||
|
||||
public func resetCoreData() {
|
||||
UserDefaults.standard.set(
|
||||
DateFormatter.formatterISO8601.string(from: Date(timeIntervalSinceReferenceDate: 0)),
|
||||
forKey: UserDefaultKey.lastItemSyncTime.rawValue
|
||||
)
|
||||
lastItemSyncTime = DateFormatter.formatterISO8601.string(from: Date(timeIntervalSinceReferenceDate: 0))
|
||||
|
||||
clearCoreData()
|
||||
|
||||
|
|
@ -177,10 +179,9 @@ public final class DataService: ObservableObject {
|
|||
linkedItem.contentReader = "PDF"
|
||||
linkedItem.tempPDFURL = localUrl
|
||||
linkedItem.title = PDFUtils.titleFromPdfFile(pageScrape.url)
|
||||
case let .html(html: html, title: title, iconURL: iconURL):
|
||||
case let .html(html: html, title: title, highlightData: _):
|
||||
linkedItem.contentReader = "WEB"
|
||||
linkedItem.originalHtml = html
|
||||
linkedItem.imageURLString = iconURL
|
||||
linkedItem.title = title ?? PDFUtils.titleFromPdfFile(pageScrape.url)
|
||||
case .none:
|
||||
linkedItem.contentReader = "WEB"
|
||||
|
|
|
|||
|
|
@ -5,9 +5,13 @@ import Utils
|
|||
|
||||
public extension DataService {
|
||||
func prefetchPages(itemIDs: [String], username: String) async {
|
||||
// TODO: make this concurrent
|
||||
for itemID in itemIDs {
|
||||
await prefetchPage(pendingLink: PendingLink(itemID: itemID, retryCount: 1), username: username)
|
||||
await withTaskGroup(of: Void.self) { group in
|
||||
for itemID in itemIDs {
|
||||
group.addTask {
|
||||
await self.prefetchPage(pendingLink: PendingLink(itemID: itemID, retryCount: 1), username: username)
|
||||
}
|
||||
}
|
||||
await group.waitForAll()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ public extension DataService {
|
|||
func syncLinkedItems(
|
||||
since date: Date,
|
||||
cursor: String?,
|
||||
previousQueryResult: LinkedItemQueryResult? = nil
|
||||
) async throws -> LinkedItemQueryResult {
|
||||
previousQueryResult: LinkedItemSyncResult? = nil
|
||||
) async throws -> LinkedItemSyncResult? {
|
||||
if previousQueryResult == nil {
|
||||
// Send offline changes to server before fetching items
|
||||
// only on the first call of this function
|
||||
|
|
@ -26,16 +26,17 @@ public extension DataService {
|
|||
|
||||
LinkedItem.deleteItems(ids: fetchResult.deletedItemIDs, context: backgroundContext)
|
||||
|
||||
guard let itemIDs = fetchResult.items.persist(context: backgroundContext) else {
|
||||
if fetchResult.items.persist(context: backgroundContext) == nil {
|
||||
throw BasicError.message(messageText: "CoreData error")
|
||||
}
|
||||
|
||||
let result = LinkedItemQueryResult(
|
||||
itemIDs: itemIDs + (previousQueryResult?.itemIDs ?? []),
|
||||
let prev = previousQueryResult?.updatedItemIDs ?? []
|
||||
let result = LinkedItemSyncResult(
|
||||
updatedItemIDs: prev + fetchResult.items.map(\.id),
|
||||
cursor: fetchResult.cursor
|
||||
)
|
||||
|
||||
if fetchResult.hasMoreItems, (previousQueryResult?.itemIDs.count ?? 0) < 200 {
|
||||
if fetchResult.hasMoreItems, (previousQueryResult?.updatedItemIDs.count ?? 0) < 200 {
|
||||
return try await syncLinkedItems(
|
||||
since: date,
|
||||
cursor: fetchResult.cursor,
|
||||
|
|
@ -58,7 +59,7 @@ public extension DataService {
|
|||
cursor: String?
|
||||
) async throws -> LinkedItemQueryResult {
|
||||
// Send offline changes to server before fetching items
|
||||
try? await syncOfflineItemsWithServerIfNeeded()
|
||||
// try? await syncOfflineItemsWithServerIfNeeded()
|
||||
|
||||
let fetchResult = try await fetchLinkedItems(limit: limit, searchQuery: searchQuery, cursor: cursor)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,13 +11,60 @@ function iconURL() {
|
|||
}
|
||||
|
||||
ShareExtension.prototype = {
|
||||
markHighlightSelection: () => {
|
||||
// First remove any previous markers, this would only normally happen during debugging
|
||||
try {
|
||||
const markers = window.document.querySelectorAll(
|
||||
`span[data-omnivore-highlight-start="true"],
|
||||
span[data-omnivore-highlight-end="true"]`
|
||||
)
|
||||
|
||||
for (let i = 0; i < markers.length; i++) {
|
||||
markers[i].remove();
|
||||
}
|
||||
} catch {
|
||||
// This should be OK
|
||||
}
|
||||
try {
|
||||
const sel = window.getSelection();
|
||||
if (sel.rangeCount) {
|
||||
const range = sel.getRangeAt(0)
|
||||
const endMarker = document.createElement("span")
|
||||
const startMarker = document.createElement("span")
|
||||
endMarker.setAttribute("data-omnivore-highlight-end", "true")
|
||||
startMarker.setAttribute("data-omnivore-highlight-start", "true")
|
||||
|
||||
var container = document.createElement("div");
|
||||
for (var i = 0, len = sel.rangeCount; i < len; ++i) {
|
||||
container.appendChild(sel.getRangeAt(i).cloneContents());
|
||||
}
|
||||
|
||||
const endRange = range.cloneRange()
|
||||
endRange.collapse(false)
|
||||
endRange.insertNode(endMarker)
|
||||
|
||||
range.insertNode(startMarker)
|
||||
|
||||
return {
|
||||
highlightHTML: container.innerHTML,
|
||||
highlightText: container.innerText
|
||||
}
|
||||
}
|
||||
} catch(error) {
|
||||
console.log("ERROR", error)
|
||||
}
|
||||
return null
|
||||
},
|
||||
run: function(arguments) {
|
||||
const highlightData = this.markHighlightSelection()
|
||||
|
||||
arguments.completionFunction({
|
||||
'url': window.location.href,
|
||||
'title': document.title.toString(),
|
||||
'iconURL': iconURL(),
|
||||
'contentType': document.contentType,
|
||||
'originalHTML': new XMLSerializer().serializeToString(document)
|
||||
'originalHTML': new XMLSerializer().serializeToString(document),
|
||||
...highlightData
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1747,6 +1747,7 @@ export type SaveError = {
|
|||
};
|
||||
|
||||
export enum SaveErrorCode {
|
||||
EmbeddedHighlightFailed = 'EMBEDDED_HIGHLIGHT_FAILED',
|
||||
Unauthorized = 'UNAUTHORIZED',
|
||||
Unknown = 'UNKNOWN'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1242,6 +1242,7 @@ type SaveError {
|
|||
}
|
||||
|
||||
enum SaveErrorCode {
|
||||
EMBEDDED_HIGHLIGHT_FAILED
|
||||
UNAUTHORIZED
|
||||
UNKNOWN
|
||||
}
|
||||
|
|
|
|||
|
|
@ -490,6 +490,7 @@ const schema = gql`
|
|||
enum SaveErrorCode {
|
||||
UNKNOWN
|
||||
UNAUTHORIZED
|
||||
EMBEDDED_HIGHLIGHT_FAILED
|
||||
}
|
||||
|
||||
type SaveError {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ import normalizeUrl from 'normalize-url'
|
|||
import { createPageSaveRequest } from './create_page_save_request'
|
||||
import { ArticleSavingRequestStatus, Page } from '../elastic/types'
|
||||
import { createPage, getPageByParam, updatePage } from '../elastic/pages'
|
||||
import { addHighlightToPage } from '../elastic/highlights'
|
||||
|
||||
type SaveContext = {
|
||||
pubsub: PubsubClient
|
||||
|
|
@ -101,12 +102,15 @@ export const savePage = async (
|
|||
savedAt: new Date(),
|
||||
}
|
||||
|
||||
let pageId: string | undefined = undefined
|
||||
const existingPage = await getPageByParam({
|
||||
userId: saver.userId,
|
||||
url: articleToSave.url,
|
||||
state: ArticleSavingRequestStatus.Succeeded,
|
||||
})
|
||||
|
||||
if (existingPage) {
|
||||
pageId = existingPage.id
|
||||
if (
|
||||
!(await updatePage(
|
||||
existingPage.id,
|
||||
|
|
@ -139,7 +143,8 @@ export const savePage = async (
|
|||
}
|
||||
}
|
||||
} else {
|
||||
if (!(await createPage(articleToSave, ctx))) {
|
||||
pageId = await createPage(articleToSave, ctx)
|
||||
if (!pageId) {
|
||||
return {
|
||||
errorCodes: [SaveErrorCode.Unknown],
|
||||
message: 'Failed to create new page',
|
||||
|
|
@ -147,6 +152,28 @@ export const savePage = async (
|
|||
}
|
||||
}
|
||||
|
||||
if (pageId && parseResult.highlightData) {
|
||||
const highlight = {
|
||||
updatedAt: new Date(),
|
||||
createdAt: new Date(),
|
||||
userId: ctx.uid,
|
||||
elasticPageId: pageId,
|
||||
...parseResult.highlightData,
|
||||
}
|
||||
|
||||
if (
|
||||
!(await addHighlightToPage(pageId, highlight, {
|
||||
pubsub: ctx.pubsub,
|
||||
uid: ctx.uid,
|
||||
}))
|
||||
) {
|
||||
return {
|
||||
errorCodes: [SaveErrorCode.EmbeddedHighlightFailed],
|
||||
message: 'Failed to save highlight',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
clientRequestId: input.clientRequestId,
|
||||
url: `${homePageURL()}/${saver.username}/${slug}`,
|
||||
|
|
|
|||
327
packages/api/src/utils/highlightGenerator.ts
Normal file
327
packages/api/src/utils/highlightGenerator.ts
Normal file
|
|
@ -0,0 +1,327 @@
|
|||
import { diff_match_patch as DiffMatchPatch } from 'diff-match-patch'
|
||||
import { interpolationSearch } from './interpolationSearch'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { nanoid } from 'nanoid'
|
||||
|
||||
const highlightTag = 'omnivore_highlight'
|
||||
export const maxHighlightLength = 2000
|
||||
|
||||
const nonParagraphTagsRegEx =
|
||||
/^(a|b|basefont|bdo|big|em|font|i|s|small|span|strike|strong|su[bp]|tt|u|code|mark)$/i
|
||||
const highlightContentRegex = new RegExp(
|
||||
`<${highlightTag}>([\\s\\S]*)<\\/${highlightTag}>`,
|
||||
'i'
|
||||
)
|
||||
const maxDeepPatchDistance = 4000
|
||||
const maxDeepPatchThreshhold = 0.5
|
||||
const maxSurroundingTextLength = 2000
|
||||
|
||||
type TextNode = {
|
||||
startIndex: number
|
||||
node: Node
|
||||
isParagraphStart?: boolean
|
||||
}
|
||||
|
||||
type ArticleTextContent = {
|
||||
textNodes: TextNode[]
|
||||
articleText: string
|
||||
}
|
||||
|
||||
export type EmbeddedHighlightData = {
|
||||
prefix: string
|
||||
suffix: string
|
||||
quote: string
|
||||
id: string
|
||||
shortId: string
|
||||
patch: string
|
||||
}
|
||||
|
||||
function getTextNodesBetween(rootNode: Node, startNode: Node, endNode: Node) {
|
||||
let textNodeStartingPoint = 0
|
||||
let articleText = ''
|
||||
let newParagraph = false
|
||||
const textNodes: TextNode[] = []
|
||||
let pastStartNode = false,
|
||||
reachedEndNode = false
|
||||
|
||||
function pushNode(node: Node) {
|
||||
textNodes.push({
|
||||
node,
|
||||
startIndex: textNodeStartingPoint,
|
||||
isParagraphStart: newParagraph,
|
||||
})
|
||||
textNodeStartingPoint += node.nodeValue?.length || 0
|
||||
articleText += node.nodeValue
|
||||
newParagraph = false
|
||||
}
|
||||
|
||||
function getTextNodes(node: Node) {
|
||||
if (node == startNode) {
|
||||
pastStartNode = true
|
||||
}
|
||||
|
||||
if (node.nodeType == 3) {
|
||||
if (
|
||||
pastStartNode &&
|
||||
!reachedEndNode &&
|
||||
!/^\s*$/.test(node.nodeValue || '')
|
||||
) {
|
||||
pushNode(node)
|
||||
}
|
||||
} else {
|
||||
if (!nonParagraphTagsRegEx.test((node as Element).tagName))
|
||||
newParagraph = true
|
||||
}
|
||||
|
||||
for (
|
||||
let i = 0, len = node.childNodes.length;
|
||||
!reachedEndNode && i < len;
|
||||
++i
|
||||
) {
|
||||
getTextNodes(node.childNodes[i])
|
||||
}
|
||||
|
||||
if (node == endNode) {
|
||||
reachedEndNode = true
|
||||
}
|
||||
}
|
||||
|
||||
getTextNodes(rootNode)
|
||||
|
||||
return {
|
||||
textNodes,
|
||||
articleText,
|
||||
}
|
||||
}
|
||||
|
||||
export const findEmbeddedHighlight = (
|
||||
dom: Element
|
||||
): EmbeddedHighlightData | undefined => {
|
||||
const startNode = dom.querySelector(
|
||||
'span[data-omnivore-highlight-start="true"]'
|
||||
)
|
||||
const endNode = dom.querySelector('span[data-omnivore-highlight-end="true"]')
|
||||
|
||||
const articleContentElement = dom
|
||||
if (!articleContentElement || !startNode || !endNode) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const beforeNodes = getTextNodesBetween(dom, articleContentElement, startNode)
|
||||
const highlightNodes = getTextNodesBetween(dom, startNode, endNode)
|
||||
const afterNodes = getTextNodesBetween(dom, endNode, articleContentElement)
|
||||
const allArticleNodes = getTextNodesBetween(
|
||||
dom,
|
||||
articleContentElement,
|
||||
articleContentElement
|
||||
)
|
||||
|
||||
const patch = generateDiffPatch(
|
||||
allArticleNodes,
|
||||
beforeNodes,
|
||||
highlightNodes,
|
||||
afterNodes
|
||||
)
|
||||
|
||||
const id = uuidv4()
|
||||
const shortId = nanoid(8)
|
||||
const info = getPrefixAndSuffix(allArticleNodes, patch)
|
||||
const quote = getQuoteText(highlightNodes)
|
||||
|
||||
return {
|
||||
id,
|
||||
shortId,
|
||||
quote,
|
||||
patch,
|
||||
prefix: info.prefix,
|
||||
suffix: info.suffix,
|
||||
}
|
||||
}
|
||||
|
||||
const getQuoteText = (highlight: ArticleTextContent): string => {
|
||||
let quote = ''
|
||||
|
||||
highlight.textNodes.forEach((textNode, i) => {
|
||||
if (textNode.isParagraphStart && i > 0) {
|
||||
quote += '\n'
|
||||
}
|
||||
quote += textNode.node.textContent
|
||||
})
|
||||
|
||||
return quote
|
||||
}
|
||||
|
||||
function generateDiffPatch(
|
||||
allArticleNodes: ArticleTextContent,
|
||||
beforeNodes: ArticleTextContent,
|
||||
highlightNodes: ArticleTextContent,
|
||||
afterNodes: ArticleTextContent
|
||||
): string {
|
||||
const textWithTags = `${beforeNodes.articleText}<${highlightTag}>${highlightNodes.articleText}</${highlightTag}>${afterNodes.articleText}`
|
||||
const diffMatchPatch = new DiffMatchPatch()
|
||||
const patch = diffMatchPatch.patch_toText(
|
||||
diffMatchPatch.patch_make(allArticleNodes.articleText, textWithTags)
|
||||
)
|
||||
|
||||
if (!patch) throw new Error('Invalid patch')
|
||||
return patch
|
||||
}
|
||||
|
||||
function getPrefixAndSuffix(
|
||||
articleTextNodes: ArticleTextContent,
|
||||
patch: string
|
||||
): {
|
||||
prefix: string
|
||||
suffix: string
|
||||
highlightTextStart: number
|
||||
highlightTextEnd: number
|
||||
textNodes: TextNode[]
|
||||
textNodeIndex: number
|
||||
} {
|
||||
if (!patch) throw new Error('Invalid patch')
|
||||
const textNodes = articleTextNodes.textNodes
|
||||
|
||||
const { highlightTextStart, highlightTextEnd } = selectionOffsetsFromPatch(
|
||||
articleTextNodes.articleText,
|
||||
patch
|
||||
)
|
||||
|
||||
// Searching for the starting text node using interpolation search algorithm
|
||||
const textNodeIndex = interpolationSearch(
|
||||
textNodes.map(({ startIndex: startIndex }) => startIndex),
|
||||
highlightTextStart
|
||||
)
|
||||
const endTextNodeIndex = interpolationSearch(
|
||||
textNodes.map(({ startIndex: startIndex }) => startIndex),
|
||||
highlightTextEnd
|
||||
)
|
||||
|
||||
const prefix = getSurroundingText({
|
||||
textNodes,
|
||||
startingTextNodeIndex: textNodeIndex,
|
||||
startingOffset: highlightTextStart - textNodes[textNodeIndex].startIndex,
|
||||
side: 'prefix',
|
||||
})
|
||||
const suffix = getSurroundingText({
|
||||
textNodes,
|
||||
startingTextNodeIndex: endTextNodeIndex,
|
||||
startingOffset: highlightTextEnd - textNodes[endTextNodeIndex].startIndex,
|
||||
side: 'suffix',
|
||||
})
|
||||
return {
|
||||
prefix,
|
||||
suffix,
|
||||
highlightTextStart,
|
||||
highlightTextEnd,
|
||||
textNodes,
|
||||
textNodeIndex,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the part of text from the starting point to the paragraph ending from the
|
||||
* specified side
|
||||
* @param param0 - Object that includes textNodes array, starting point and the
|
||||
* way of movement (prefix, suffix)
|
||||
* @returns String of text to fulfill the paragraph that surrounds the
|
||||
* highlight from either starting or ending point
|
||||
*/
|
||||
const getSurroundingText = ({
|
||||
textNodes,
|
||||
startingTextNodeIndex,
|
||||
startingOffset,
|
||||
side,
|
||||
}: {
|
||||
textNodes: TextNode[]
|
||||
startingTextNodeIndex: number
|
||||
startingOffset: number
|
||||
side: 'prefix' | 'suffix'
|
||||
}): string => {
|
||||
const isPrefix = side === 'prefix'
|
||||
let i = startingTextNodeIndex
|
||||
const getTextPart = (): string => {
|
||||
i += isPrefix ? -1 : 1
|
||||
const { node, isParagraphStart: startsParagraph } = textNodes[i]
|
||||
const text = node.nodeValue || ''
|
||||
|
||||
if (isPrefix) {
|
||||
if (startsParagraph) return text
|
||||
if (text.length > maxSurroundingTextLength) return text
|
||||
return getTextPart() + text
|
||||
} else {
|
||||
if (!textNodes[i + 1] || textNodes[i + 1].isParagraphStart) return text
|
||||
if (text.length > maxSurroundingTextLength) return text
|
||||
return text + getTextPart()
|
||||
}
|
||||
}
|
||||
const truncateText = (str: string): string => {
|
||||
if (str.length <= maxSurroundingTextLength) return str
|
||||
if (isPrefix) {
|
||||
return str.slice(str.length - maxSurroundingTextLength)
|
||||
}
|
||||
return str.substring(0, maxSurroundingTextLength)
|
||||
}
|
||||
|
||||
const { isParagraphStart: startsParagraph, node } =
|
||||
textNodes[startingTextNodeIndex]
|
||||
const nodeText = node.nodeValue || ''
|
||||
|
||||
const text = isPrefix
|
||||
? nodeText.substring(0, startingOffset)
|
||||
: nodeText.substring(startingOffset)
|
||||
|
||||
if (isPrefix) {
|
||||
return truncateText(startsParagraph ? text : getTextPart() + text)
|
||||
} else {
|
||||
return truncateText(
|
||||
!textNodes[i + 1] || textNodes[i + 1].isParagraphStart
|
||||
? text
|
||||
: text + getTextPart()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const selectionOffsetsFromPatch = (
|
||||
articleText: string,
|
||||
patch: string
|
||||
): {
|
||||
highlightTextStart: number
|
||||
highlightTextEnd: number
|
||||
matchingHighlightContent: RegExpExecArray
|
||||
} => {
|
||||
if (!patch) throw new Error('Invalid patch')
|
||||
const dmp = new DiffMatchPatch()
|
||||
// Applying a patch to the whole article text to find the selection content via regexp
|
||||
const appliedPatch = dmp.patch_apply(dmp.patch_fromText(patch), articleText)
|
||||
|
||||
let matchingHighlightContent
|
||||
if (!appliedPatch[1][0]) {
|
||||
dmp.Match_Threshold = maxDeepPatchThreshhold
|
||||
dmp.Match_Distance = maxDeepPatchDistance
|
||||
const deeperAppliedPatch = dmp.patch_apply(
|
||||
dmp.patch_fromText(patch),
|
||||
articleText
|
||||
)
|
||||
if (!deeperAppliedPatch[1][0]) {
|
||||
throw new Error('Unable to find the highlight')
|
||||
} else {
|
||||
matchingHighlightContent = highlightContentRegex.exec(
|
||||
deeperAppliedPatch[0]
|
||||
)
|
||||
}
|
||||
} else {
|
||||
matchingHighlightContent = highlightContentRegex.exec(appliedPatch[0])
|
||||
}
|
||||
|
||||
if (!matchingHighlightContent)
|
||||
throw new Error('Unable to find the highlight from patch')
|
||||
|
||||
const highlightTextStart = matchingHighlightContent.index
|
||||
const highlightTextEnd =
|
||||
highlightTextStart + matchingHighlightContent[1].length
|
||||
return {
|
||||
highlightTextStart,
|
||||
highlightTextEnd,
|
||||
matchingHighlightContent,
|
||||
}
|
||||
}
|
||||
32
packages/api/src/utils/interpolationSearch.ts
Normal file
32
packages/api/src/utils/interpolationSearch.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* Finds the index of the element which value is the closest to the target
|
||||
* @param arr - An array of numbers to search from
|
||||
* @param target - The target number to find
|
||||
* @returns The index of the closest to the target value array element
|
||||
*/
|
||||
|
||||
export function interpolationSearch(arr: number[], target: number): number {
|
||||
let left = 0
|
||||
let right = arr.length - 1
|
||||
while (left < right) {
|
||||
const rangeDelta = arr[right] - arr[left]
|
||||
const indexDelta = right - left
|
||||
const valueDelta = target - arr[left]
|
||||
if (valueDelta < 0) {
|
||||
throw new Error('Unable to find text node')
|
||||
}
|
||||
if (!rangeDelta) {
|
||||
return left
|
||||
}
|
||||
const middleIndex =
|
||||
left + Math.floor((valueDelta * indexDelta) / rangeDelta)
|
||||
if (target < arr[middleIndex]) {
|
||||
right = middleIndex
|
||||
} else if (target >= arr[middleIndex + 1]) {
|
||||
left = middleIndex + 1
|
||||
} else {
|
||||
return middleIndex
|
||||
}
|
||||
}
|
||||
throw new Error('Unable to find text node')
|
||||
}
|
||||
|
|
@ -16,6 +16,10 @@ import { ILike } from 'typeorm'
|
|||
import { v4 as uuid } from 'uuid'
|
||||
import addressparser from 'addressparser'
|
||||
import { preParseContent } from '@omnivore/content-handler'
|
||||
import {
|
||||
findEmbeddedHighlight,
|
||||
EmbeddedHighlightData,
|
||||
} from './highlightGenerator'
|
||||
|
||||
const logger = buildLogger('utils.parse')
|
||||
|
||||
|
|
@ -70,6 +74,7 @@ export type ParsedContentPuppeteer = {
|
|||
parsedContent: Readability.ParseResult | null
|
||||
canonicalUrl?: string | null
|
||||
pageType: PageType
|
||||
highlightData?: EmbeddedHighlightData
|
||||
}
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
|
|
@ -178,6 +183,7 @@ export const parsePreparedContent = async (
|
|||
}
|
||||
|
||||
let article = null
|
||||
let highlightData = undefined
|
||||
const { document, pageInfo } = preparedDocument
|
||||
|
||||
// Checking for content type acceptance or if there are no contentType
|
||||
|
|
@ -234,6 +240,10 @@ export const parsePreparedContent = async (
|
|||
article.content = article.dom.outerHTML
|
||||
}
|
||||
|
||||
if (article?.dom) {
|
||||
highlightData = findEmbeddedHighlight(article?.dom)
|
||||
}
|
||||
|
||||
const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
|
||||
'omnivore-highlight-id',
|
||||
'data-twitter-tweet-id',
|
||||
|
|
@ -315,6 +325,7 @@ export const parsePreparedContent = async (
|
|||
parsedContent: article,
|
||||
canonicalUrl,
|
||||
pageType: parseOriginalContent(dom),
|
||||
highlightData,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { createTestUser, deleteTestUser } from '../db'
|
|||
import { SaveContext, saveEmail } from '../../src/services/save_email'
|
||||
import { createPubSubClient } from '../../src/datalayer/pubsub'
|
||||
import { getPageByParam } from '../../src/elastic/pages'
|
||||
import nock from 'nock'
|
||||
|
||||
describe('saveEmail', () => {
|
||||
const username = 'fakeUser'
|
||||
|
|
@ -15,6 +16,8 @@ describe('saveEmail', () => {
|
|||
})
|
||||
|
||||
it('doesnt fail if saved twice', async () => {
|
||||
nock('https://blog.omnivore.app').get('/fake-url').reply(404)
|
||||
|
||||
const url = 'https://blog.omnivore.app/fake-url'
|
||||
const title = 'fake title'
|
||||
const author = 'fake author'
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import { NewsletterEmail } from '../../src/entity/newsletter_email'
|
|||
import { SaveContext } from '../../src/services/save_email'
|
||||
import { createPubSubClient } from '../../src/datalayer/pubsub'
|
||||
import { getPageByParam } from '../../src/elastic/pages'
|
||||
import nock from 'nock'
|
||||
|
||||
describe('saveNewsletterEmail', () => {
|
||||
const username = 'fakeUser'
|
||||
|
|
@ -35,6 +36,7 @@ describe('saveNewsletterEmail', () => {
|
|||
})
|
||||
|
||||
it('adds the newsletter to the library', async () => {
|
||||
nock('https://blog.omnivore.app').get('/fake-url').reply(404)
|
||||
const url = 'https://blog.omnivore.app/fake-url'
|
||||
|
||||
await saveNewsletterEmail(
|
||||
|
|
|
|||
|
|
@ -38,9 +38,9 @@ describe('parseMetadata', async () => {
|
|||
|
||||
describe('parsePreparedContent', async () => {
|
||||
it('gets published date when JSONLD fails to load', async () => {
|
||||
nock('https://stratechery.com:443', {"encodedQueryParams":true})
|
||||
nock('https://stratechery.com:443', { encodedQueryParams: true })
|
||||
.get('/wp-json/oembed/1.0/embed')
|
||||
.query({"url":"https%3A%2F%2Fstratechery.com%2F2016%2Fits-a-tesla%2F"})
|
||||
.query({ url: 'https%3A%2F%2Fstratechery.com%2F2016%2Fits-a-tesla%2F' })
|
||||
.reply(401)
|
||||
|
||||
const html = load('./test/utils/data/stratechery-blog-post.html')
|
||||
|
|
@ -53,6 +53,29 @@ describe('parsePreparedContent', async () => {
|
|||
new Date('2016-04-05T15:27:51+00:00').getTime()
|
||||
)
|
||||
})
|
||||
it('returns a highlight range if markers are found in the HTML', async () => {
|
||||
const html = `
|
||||
<html>
|
||||
<body>
|
||||
<div>
|
||||
<div id='article-container'>
|
||||
some prefix text
|
||||
<span data-omnivore-highlight-start="true"></span>This is some text within the highlight markers<span data-omnivore-highlight-end="true"></span>
|
||||
some suffix text
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
`
|
||||
const result = await parsePreparedContent('https://blog.omnivore.app/', {
|
||||
document: html,
|
||||
pageInfo: {},
|
||||
})
|
||||
|
||||
expect(result.highlightData?.quote).to.eq(
|
||||
'This is some text within the highlight markers'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('parsePreparedContent', async () => {
|
||||
|
|
|
|||
8
packages/rule-handler/test/stub.test.ts
Normal file
8
packages/rule-handler/test/stub.test.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
import 'mocha'
|
||||
import { expect } from 'chai'
|
||||
|
||||
describe('stub test', () => {
|
||||
it('should pass', () => {
|
||||
expect(true).to.be.true
|
||||
})
|
||||
})
|
||||
Loading…
Reference in a new issue