Add a refresh context to make debugging rss jobs easier

packages/api/src/jobs/rss/refreshAllFeeds.ts

@@ -5,8 +5,20 @@ import { redisDataSource } from '../../redis_data_source'
 import { RssSubscriptionGroup } from '../../utils/createTask'
 import { stringToHash } from '../../utils/helpers'
 import { validateUrl } from '../../services/create_page_save_request'
+import { v4 as uuid } from 'uuid'
+
+type RSSRefreshContext = {
+  type: 'all' | 'user-added'
+  refreshID: string
+  startedAt: string
+}
 
 export const refreshAllFeeds = async (db: DataSource): Promise<boolean> => {
+  const refreshContext = {
+    type: 'all',
+    refreshID: uuid(),
+    startedAt: new Date().toISOString(),
+  } as RSSRefreshContext
   const subscriptionGroups = (await db.createEntityManager().query(
     `
     SELECT
@@ -30,19 +42,27 @@ export const refreshAllFeeds = async (db: DataSource): Promise<boolean> => {
     ['RSS', 'ACTIVE', 'following']
   )) as RssSubscriptionGroup[]
 
+  console.log(`rss: checking ${subscriptionGroups.length}`, { refreshContext })
+
   for (const group of subscriptionGroups) {
     try {
-      await updateSubscriptionGroup(group)
+      await updateSubscriptionGroup(group, refreshContext)
     } catch (err) {
       // we don't want to fail the whole job if one subscription group fails
       console.error('error updating subscription group')
     }
   }
+  console.log(`rss: finished queuing subscription groups at ${new Date()}`, {
+    refreshContext,
+  })
 
   return true
 }
 
-const updateSubscriptionGroup = async (group: RssSubscriptionGroup) => {
+const updateSubscriptionGroup = async (
+  group: RssSubscriptionGroup,
+  refreshContext: RSSRefreshContext
+) => {
   let feedURL = group.url
   const userList = JSON.stringify(group.userIds.sort())
   if (!feedURL) {
@@ -63,6 +83,7 @@ const updateSubscriptionGroup = async (group: RssSubscriptionGroup) => {
     userList
   )}`
   const payload = {
+    refreshContext,
     subscriptionIds: group.subscriptionIds,
     feedUrl: group.url,
     lastFetchedTimestamps: group.fetchedDates.map(

packages/api/src/utils/createTask.ts

@@ -22,6 +22,7 @@ import View = google.cloud.tasks.v2.Task.View
 import { stringToHash } from './helpers'
 import { queueRSSRefreshFeedJob } from '../jobs/rss/refreshAllFeeds'
 import { redisDataSource } from '../redis_data_source'
+import { v4 as uuid } from 'uuid'
 
 // Instantiates a client.
 const client = new CloudTasksClient()
@@ -639,8 +640,12 @@ export interface RssSubscriptionGroup {
 export const enqueueRssFeedFetch = async (
   subscriptionGroup: RssSubscriptionGroup
 ): Promise<string> => {
-  const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env
   const payload = {
+    refreshContext: {
+      type: 'user-added',
+      refreshID: uuid(),
+      startedAt: new Date().toISOString(),
+    },
     subscriptionIds: subscriptionGroup.subscriptionIds,
     feedUrl: subscriptionGroup.url,
     lastFetchedTimestamps: subscriptionGroup.fetchedDates.map(
@@ -670,40 +675,6 @@ export const enqueueRssFeedFetch = async (
   } else {
     throw 'unable to queue rss-refresh-feed-job, redis is not configured'
   }
-
-  // // If there is no Google Cloud Project Id exposed, it means that we are in local environment
-  // if (env.dev.isLocal || !GOOGLE_CLOUD_PROJECT) {
-  //   if (env.queue.rssFeedTaskHandlerUrl) {
-  //     // Calling the handler function directly.
-  //     setTimeout(() => {
-  //       axios
-  //         .post(
-  //           `${env.queue.rssFeedTaskHandlerUrl}?token=${PUBSUB_VERIFICATION_TOKEN}`,
-  //           payload
-  //         )
-  //         .catch((error) => {
-  //           logError(error)
-  //         })
-  //     }, 0)
-  //   }
-  //   return nanoid()
-  // }
-
-  // const createdTasks = await createHttpTaskWithToken({
-  //   project: GOOGLE_CLOUD_PROJECT,
-  //   queue: 'omnivore-rss-queue',
-  //   payload,
-  //   taskHandlerUrl: `${env.queue.rssFeedTaskHandlerUrl}?token=${PUBSUB_VERIFICATION_TOKEN}`,
-  // })
-
-  // if (!createdTasks || !createdTasks[0].name) {
-  //   logger.error(`Unable to get the name of the task`, {
-  //     payload,
-  //     createdTasks,
-  //   })
-  //   throw new CreateTaskError(`Unable to get the name of the task`)
-  // }
-  //return createdTasks[0].name
 }
 
 export default createHttpTaskWithToken