From 3cdf7b47f6fb0b5f0241d8260b78f6952e05a9a8 Mon Sep 17 00:00:00 2001 From: Satindar Dhillon Date: Wed, 18 May 2022 14:08:37 -0700 Subject: [PATCH 1/5] enable background fetch in plist --- apple/InfoPlists/Omnivore.plist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apple/InfoPlists/Omnivore.plist b/apple/InfoPlists/Omnivore.plist index e9315d58e..354303591 100644 --- a/apple/InfoPlists/Omnivore.plist +++ b/apple/InfoPlists/Omnivore.plist @@ -48,6 +48,10 @@ Images from documents and annotations can be saved to the photo library. NSPhotoLibraryUsageDescription Images from your photo library can be chosen by you to send as feedback. + UIBackgroundModes + + fetch + UILaunchStoryboardName LaunchScreen UIMainStoryboardFile From 801b17425461607b74b8ef8c380a7fe0d7082f29 Mon Sep 17 00:00:00 2001 From: Satindar Dhillon Date: Wed, 18 May 2022 15:52:13 -0700 Subject: [PATCH 2/5] implement background task functions in services class --- apple/InfoPlists/Omnivore.plist | 4 ++ apple/OmnivoreKit/Sources/App/Services.swift | 55 +++++++++++++++++++ .../Sources/App/Views/RootView/RootView.swift | 6 ++ apple/Sources/AppDelegate.swift | 11 ++-- 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/apple/InfoPlists/Omnivore.plist b/apple/InfoPlists/Omnivore.plist index 354303591..947099c90 100644 --- a/apple/InfoPlists/Omnivore.plist +++ b/apple/InfoPlists/Omnivore.plist @@ -2,6 +2,10 @@ + BGTaskSchedulerPermittedIdentifiers + + app.omnivore.fetchLinkedItems + CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleDisplayName diff --git a/apple/OmnivoreKit/Sources/App/Services.swift b/apple/OmnivoreKit/Sources/App/Services.swift index c73ff44bb..99e5b8425 100644 --- a/apple/OmnivoreKit/Sources/App/Services.swift +++ b/apple/OmnivoreKit/Sources/App/Services.swift @@ -1,8 +1,14 @@ +import BackgroundTasks import Foundation import Models +import OSLog import Services public final class Services { + static let fetchTaskID = "app.omnivore.fetchLinkedItems" + static let secondsToWaitBeforeNextBackgroundRefresh: TimeInterval = 3 // 1 hour + static let logger = Logger(subsystem: "app.omnivore", category: "services-class") + public let authenticator: Authenticator public let dataService: DataService @@ -12,3 +18,52 @@ public final class Services { self.dataService = DataService(appEnvironment: appEnvironment, networker: networker) } } + +// Background fetching functions +extension Services { + public static func registerBackgroundFetch() { + BGTaskScheduler.shared.register(forTaskWithIdentifier: fetchTaskID, using: nil) { task in + if let task = task as? BGAppRefreshTask { + startBackgroundFetch(task: task) + } + } + } + + static func scheduleBackgroundFetch() { + let taskRequest = BGProcessingTaskRequest(identifier: fetchTaskID) + taskRequest.requiresNetworkConnectivity = true + taskRequest.earliestBeginDate = Date(timeIntervalSinceNow: secondsToWaitBeforeNextBackgroundRefresh) + + do { + try BGTaskScheduler.shared.submit(taskRequest) + logger.debug("\(fetchTaskID) task scheduled") + } catch { + logger.debug("task scheduling failed: \(fetchTaskID)") + } + } + + static func startBackgroundFetch(task: BGAppRefreshTask) { + scheduleBackgroundFetch() + let services = Services() + + task.expirationHandler = { + logger.debug("handling background fetch expiration") + // cancel tasks if still running + // maybe save/cancel pending changes to coredata context? + } + + services.peformBackgroundFetch() + } + + func peformBackgroundFetch() { + Services.logger.debug("starting background fetch") + + guard authenticator.hasValidAuthToken else { + Services.logger.debug("background fetch failed: user does not habe a valid auth token") + authenticator.logout() + return + } + + // TODO: perform tasks using data service + } +} diff --git a/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift b/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift index 7dee98997..4261c20a1 100644 --- a/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/RootView/RootView.swift @@ -5,6 +5,7 @@ import Utils import Views public struct RootView: View { + @Environment(\.scenePhase) var scenePhase let pdfViewerProvider: ((URL, PDFViewerViewModel) -> AnyView)? @StateObject private var viewModel = RootViewModel() @@ -31,6 +32,11 @@ public struct RootView: View { viewModel.configurePDFProvider(pdfViewerProvider: pdfViewerProvider) } } + .onChange(of: scenePhase) { phase in + if phase == .background { + Services.scheduleBackgroundFetch() + } + } } } diff --git a/apple/Sources/AppDelegate.swift b/apple/Sources/AppDelegate.swift index 65779dba9..c1c7875aa 100644 --- a/apple/Sources/AppDelegate.swift +++ b/apple/Sources/AppDelegate.swift @@ -1,12 +1,17 @@ +import OSLog + #if os(macOS) import AppKit #elseif os(iOS) + import App import Intercom import UIKit import Utils import Views #endif +private let logger = Logger(subsystem: "app.omnivore", category: "app-delegate") + #if os(macOS) class AppDelegate: NSObject, NSApplicationDelegate { func applicationDidFinishLaunching(_: Notification) { @@ -20,11 +25,6 @@ #elseif os(iOS) class AppDelegate: NSObject, UIApplicationDelegate { -// override init() { -// super.init() -// UIColor.classInit -// } - // swiftlint:disable:next line_length func application(_: UIApplication, didFinishLaunchingWithOptions _: [UIApplication.LaunchOptionsKey: Any]? = nil) -> Bool { #if DEBUG @@ -45,6 +45,7 @@ } } + Services.registerBackgroundFetch() configurePushNotifications() return true } From 64b6f95a1532676fb132f4ec37d2186396ec0596 Mon Sep 17 00:00:00 2001 From: Satindar Dhillon Date: Wed, 18 May 2022 21:36:37 -0700 Subject: [PATCH 3/5] use correct task request for background fetching --- apple/OmnivoreKit/Sources/App/Services.swift | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/apple/OmnivoreKit/Sources/App/Services.swift b/apple/OmnivoreKit/Sources/App/Services.swift index 99e5b8425..280eb6207 100644 --- a/apple/OmnivoreKit/Sources/App/Services.swift +++ b/apple/OmnivoreKit/Sources/App/Services.swift @@ -24,14 +24,15 @@ extension Services { public static func registerBackgroundFetch() { BGTaskScheduler.shared.register(forTaskWithIdentifier: fetchTaskID, using: nil) { task in if let task = task as? BGAppRefreshTask { + logger.debug("in background task register closure") startBackgroundFetch(task: task) } } } static func scheduleBackgroundFetch() { - let taskRequest = BGProcessingTaskRequest(identifier: fetchTaskID) - taskRequest.requiresNetworkConnectivity = true + BGTaskScheduler.shared.cancelAllTaskRequests() + let taskRequest = BGAppRefreshTaskRequest(identifier: fetchTaskID) taskRequest.earliestBeginDate = Date(timeIntervalSinceNow: secondsToWaitBeforeNextBackgroundRefresh) do { @@ -53,17 +54,20 @@ extension Services { } services.peformBackgroundFetch() + task.setTaskCompleted(success: true) } - + func peformBackgroundFetch() { Services.logger.debug("starting background fetch") guard authenticator.hasValidAuthToken else { Services.logger.debug("background fetch failed: user does not habe a valid auth token") - authenticator.logout() return } + Services.logger.debug("ayo this is a background task!") // TODO: perform tasks using data service } } + +// e -l objc -- (void)[[BGTaskScheduler sharedScheduler] _simulateLaunchForTaskWithIdentifier:@"app.omnivore.fetchLinkedItems"] From 2eeec7cc776e0a4eb96b3bc071936632bdb37783 Mon Sep 17 00:00:00 2001 From: Satindar Dhillon Date: Thu, 19 May 2022 13:04:05 -0700 Subject: [PATCH 4/5] send debug messages to segment for bg task execution --- apple/OmnivoreKit/Sources/App/Services.swift | 9 ++++++--- .../Sources/Utils/EventTracking/EventTracker.swift | 6 ++++++ .../Sources/Utils/EventTracking/TrackableEvents.swift | 5 +++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/apple/OmnivoreKit/Sources/App/Services.swift b/apple/OmnivoreKit/Sources/App/Services.swift index 280eb6207..46f9dd7de 100644 --- a/apple/OmnivoreKit/Sources/App/Services.swift +++ b/apple/OmnivoreKit/Sources/App/Services.swift @@ -3,10 +3,11 @@ import Foundation import Models import OSLog import Services +import Utils public final class Services { static let fetchTaskID = "app.omnivore.fetchLinkedItems" - static let secondsToWaitBeforeNextBackgroundRefresh: TimeInterval = 3 // 1 hour + static let secondsToWaitBeforeNextBackgroundRefresh: TimeInterval = isDebug ? 0 : 3600 // 1 hour static let logger = Logger(subsystem: "app.omnivore", category: "services-class") public let authenticator: Authenticator @@ -24,6 +25,7 @@ extension Services { public static func registerBackgroundFetch() { BGTaskScheduler.shared.register(forTaskWithIdentifier: fetchTaskID, using: nil) { task in if let task = task as? BGAppRefreshTask { + EventTracker.trackForDebugging("executing app.omnivore.fetchLinkedItems bg task") logger.debug("in background task register closure") startBackgroundFetch(task: task) } @@ -48,12 +50,12 @@ extension Services { let services = Services() task.expirationHandler = { + EventTracker.trackForDebugging("background fetch expiration handler called") logger.debug("handling background fetch expiration") - // cancel tasks if still running - // maybe save/cancel pending changes to coredata context? } services.peformBackgroundFetch() + EventTracker.trackForDebugging("background fetch task completed successfully") task.setTaskCompleted(success: true) } @@ -61,6 +63,7 @@ extension Services { Services.logger.debug("starting background fetch") guard authenticator.hasValidAuthToken else { + EventTracker.trackForDebugging("background fetch failed: user does not habe a valid auth token") Services.logger.debug("background fetch failed: user does not habe a valid auth token") return } diff --git a/apple/OmnivoreKit/Sources/Utils/EventTracking/EventTracker.swift b/apple/OmnivoreKit/Sources/Utils/EventTracking/EventTracker.swift index fcc5f3159..fa63eb75b 100644 --- a/apple/OmnivoreKit/Sources/Utils/EventTracking/EventTracker.swift +++ b/apple/OmnivoreKit/Sources/Utils/EventTracking/EventTracker.swift @@ -7,6 +7,12 @@ public enum EventTracker { _ = segment?.version() } + public static func trackForDebugging(_ message: String) { + #if DEBUG + track(.debugMessage(message: message)) + #endif + } + public static func track(_ event: TrackableEvent) { segment?.track(name: event.name, properties: event.properties) } diff --git a/apple/OmnivoreKit/Sources/Utils/EventTracking/TrackableEvents.swift b/apple/OmnivoreKit/Sources/Utils/EventTracking/TrackableEvents.swift index b9f019961..3450f1fca 100644 --- a/apple/OmnivoreKit/Sources/Utils/EventTracking/TrackableEvents.swift +++ b/apple/OmnivoreKit/Sources/Utils/EventTracking/TrackableEvents.swift @@ -2,6 +2,7 @@ import Foundation public enum TrackableEvent { case linkRead(linkID: String, slug: String, originalArticleURL: String) + case debugMessage(message: String) } public extension TrackableEvent { @@ -9,6 +10,8 @@ public extension TrackableEvent { switch self { case .linkRead: return "link_read" + case .debugMessage: + return "debug_message" } } @@ -20,6 +23,8 @@ public extension TrackableEvent { "slug": slug, "url": originalArticleURL ] + case let .debugMessage(message: message): + return ["message": message] } } } From d6f9acca1149a1d29297c4338ffec311923a9735 Mon Sep 17 00:00:00 2001 From: Satindar Dhillon Date: Thu, 19 May 2022 16:12:53 -0700 Subject: [PATCH 5/5] fetch items in background task --- apple/OmnivoreKit/Sources/App/Services.swift | 32 +++++---- .../App/Views/Home/HomeFeedViewModel.swift | 4 +- .../FetchLinkedItemsBackgroundTask.swift | 57 ++++++++++++++++ .../Queries/ArticleContentQuery.swift | 11 ++- .../Queries/LinkedItemIDsQuery.swift | 67 +++++++++++++++++++ .../InternalModels/InternalLinkedItem.swift | 1 - 6 files changed, 150 insertions(+), 22 deletions(-) create mode 100644 apple/OmnivoreKit/Sources/Services/DataService/FetchLinkedItemsBackgroundTask.swift create mode 100644 apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemIDsQuery.swift diff --git a/apple/OmnivoreKit/Sources/App/Services.swift b/apple/OmnivoreKit/Sources/App/Services.swift index 46f9dd7de..c12951763 100644 --- a/apple/OmnivoreKit/Sources/App/Services.swift +++ b/apple/OmnivoreKit/Sources/App/Services.swift @@ -27,7 +27,7 @@ extension Services { if let task = task as? BGAppRefreshTask { EventTracker.trackForDebugging("executing app.omnivore.fetchLinkedItems bg task") logger.debug("in background task register closure") - startBackgroundFetch(task: task) + performBackgroundFetch(task: task) } } } @@ -45,7 +45,8 @@ extension Services { } } - static func startBackgroundFetch(task: BGAppRefreshTask) { + static func performBackgroundFetch(task: BGAppRefreshTask) { + Services.logger.debug("starting background fetch") scheduleBackgroundFetch() let services = Services() @@ -54,22 +55,25 @@ extension Services { logger.debug("handling background fetch expiration") } - services.peformBackgroundFetch() - EventTracker.trackForDebugging("background fetch task completed successfully") - task.setTaskCompleted(success: true) - } - - func peformBackgroundFetch() { - Services.logger.debug("starting background fetch") - - guard authenticator.hasValidAuthToken else { - EventTracker.trackForDebugging("background fetch failed: user does not habe a valid auth token") + guard services.authenticator.hasValidAuthToken else { + EventTracker.trackForDebugging("background fetch failed: user does not have a valid auth token") Services.logger.debug("background fetch failed: user does not habe a valid auth token") + task.setTaskCompleted(success: false) return } - Services.logger.debug("ayo this is a background task!") - // TODO: perform tasks using data service + Task { + do { + try await services.dataService.fetchLinkedItemsBackgroundTask() + logger.debug("fetch complete") + EventTracker.trackForDebugging("background fetch task completed successfully") + task.setTaskCompleted(success: true) + } catch { + logger.debug("fetch failed") + EventTracker.trackForDebugging("background fetch task failed") + task.setTaskCompleted(success: false) + } + } } } diff --git a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift index a099b971f..c11d66d2c 100644 --- a/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift +++ b/apple/OmnivoreKit/Sources/App/Views/Home/HomeFeedViewModel.swift @@ -98,7 +98,9 @@ import Views isLoading = false receivedIdx = thisSearchIdx cursor = queryResult.cursor - await dataService.prefetchPages(itemIDs: newItems.map(\.unwrappedID)) + if let username = dataService.currentViewer?.username { + await dataService.prefetchPages(itemIDs: newItems.map(\.unwrappedID), username: username) + } } else { updateFetchController(dataService: dataService) } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/FetchLinkedItemsBackgroundTask.swift b/apple/OmnivoreKit/Sources/Services/DataService/FetchLinkedItemsBackgroundTask.swift new file mode 100644 index 000000000..939f3330e --- /dev/null +++ b/apple/OmnivoreKit/Sources/Services/DataService/FetchLinkedItemsBackgroundTask.swift @@ -0,0 +1,57 @@ +import CoreData +import Foundation +import Models +import SwiftGraphQL + +extension DataService { + public func fetchLinkedItemsBackgroundTask() async throws { + // Query the server for item IDs and compare against CoreData + // to see what we're missing + let missingItemIds = try await fetchMissingItemIDs() + guard !missingItemIds.isEmpty else { return } + + let username: String? = await backgroundContext.perform(schedule: .immediate) { + let fetchRequest: NSFetchRequest = Viewer.fetchRequest() + fetchRequest.fetchLimit = 1 // we should only have one viewer saved + return try? self.backgroundContext.fetch(fetchRequest).first?.username + } + + guard let username = username else { + throw BasicError.message(messageText: "could not retrieve username from core data") + } + + // Fetch the items + for itemID in missingItemIds { // TOOD: run these in parallel + logger.debug("fetching item with ID: \(itemID)") + _ = try await articleContent(username: username, itemID: itemID, useCache: false) + logger.debug("done fetching item with ID: \(itemID)") + } + } + + func fetchMissingItemIDs(previouslyFetchedIDs: [String] = [], cursor: String? = nil) async throws -> [String] { + logger.debug("fetching more IDS: \(cursor ?? "no cursor")") + let maxItemCount = 30 + let fetchResult = try await fetchLinkedItemIDs(limit: 10, cursor: cursor) + let newItemsToFetch = await itemsNotInStore(from: fetchResult.itemIDs) + let itemsToFetch = previouslyFetchedIDs + newItemsToFetch + + if newItemsToFetch.isEmpty || itemsToFetch.count > maxItemCount || fetchResult.cursor == nil { + return itemsToFetch + } else { + return try await fetchMissingItemIDs(previouslyFetchedIDs: itemsToFetch, cursor: fetchResult.cursor) + } + } + + func itemsNotInStore(from itemIDs: [String]) async -> [String] { + let fetchRequest: NSFetchRequest = LinkedItem.fetchRequest() + fetchRequest.predicate = NSPredicate(format: "id IN %@", itemIDs) + + return await backgroundContext.perform(schedule: .immediate) { + if let foundIDs = (try? self.backgroundContext.fetch(fetchRequest).map(\.unwrappedID)) { + return itemIDs.filter { !foundIDs.contains($0) } + } else { + return [] + } + } + } +} diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift index bbea73ead..ebb8a3bf8 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift @@ -9,9 +9,9 @@ extension DataService { let retryCount: Int } - public func prefetchPages(itemIDs: [String]) async { - guard let username = currentViewer?.username else { return } - + public func prefetchPages(itemIDs: [String], username: String) async { + // TODO: make this concurrent + // TODO: make a non-pending page option for BG tasks for itemID in itemIDs { await prefetchPage(pendingLink: PendingLink(itemID: itemID, retryCount: 1), username: username) } @@ -183,9 +183,8 @@ extension DataService { let fetchRequest: NSFetchRequest = LinkedItem.fetchRequest() fetchRequest.predicate = NSPredicate(format: "id == %@", item.id) - let linkedItem = try? self.backgroundContext.fetch(fetchRequest).first - - guard let linkedItem = linkedItem else { return } + let existingItem = try? self.backgroundContext.fetch(fetchRequest).first + let linkedItem = existingItem ?? LinkedItem(entity: LinkedItem.entity(), insertInto: self.backgroundContext) let highlightObjects = highlights.map { $0.asManagedObject(context: self.backgroundContext) diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemIDsQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemIDsQuery.swift new file mode 100644 index 000000000..f6dede0d3 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemIDsQuery.swift @@ -0,0 +1,67 @@ +import Foundation +import Models +import SwiftGraphQL + +extension DataService { + struct LinkedItemIDFetchResult { + let itemIDs: [String] + let cursor: String? + } + + func fetchLinkedItemIDs(limit: Int, cursor: String?) async throws -> LinkedItemIDFetchResult { + enum QueryResult { + case success(result: LinkedItemIDFetchResult) + case error(error: String) + } + + let articleIDSelection = Selection.SearchItemEdge { + try $0.node(selection: Selection.SearchItem { try $0.id() }) + } + + let selection = Selection { + try $0.on( + searchError: .init { + QueryResult.error(error: try $0.errorCodes().description) + }, + searchSuccess: .init { + QueryResult.success( + result: LinkedItemIDFetchResult( + itemIDs: try $0.edges(selection: articleIDSelection.list), + cursor: try $0.pageInfo(selection: Selection.PageInfo { + try $0.endCursor() + }) + ) + ) + } + ) + } + + let query = Selection.Query { + try $0.search( + after: OptionalArgument(cursor), + first: OptionalArgument(limit), + query: OptionalArgument(nil), + selection: selection + ) + } + + let path = appEnvironment.graphqlPath + let headers = networker.defaultHeaders + + return try await withCheckedThrowingContinuation { continuation in + send(query, to: path, headers: headers) { queryResult in + guard let payload = try? queryResult.get() else { + continuation.resume(throwing: BasicError.message(messageText: "network error")) + return + } + + switch payload.data { + case let .success(result: result): + continuation.resume(returning: result) + case .error: + continuation.resume(throwing: BasicError.message(messageText: "LinkedItem fetch error")) + } + } + } + } +} diff --git a/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift b/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift index 7214897c9..5e39aa6b9 100644 --- a/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift +++ b/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift @@ -54,7 +54,6 @@ struct InternalLinkedItem { } extension Sequence where Element == InternalLinkedItem { - // TODO: -optimization use batch update? func persist(context: NSManagedObjectContext) -> [LinkedItem]? { var linkedItems: [LinkedItem]? context.performAndWait {