check is_fetching_content and only fetch content if required for rss feed items

packages/api/src/entity/library_item.ts

@@ -196,4 +196,25 @@ export class LibraryItem {
 
   @Column('text', { nullable: true })
   recommenderNames?: string[] | null
+
+  @Column('timestamptz')
+  hiddenAt?: Date | null
+
+  @Column('timestamptz')
+  sharedAt?: Date | null
+
+  @Column('text')
+  sharedBy?: string | null
+
+  @Column('jsonb')
+  links?: Record<string, unknown>[] | null
+
+  @Column('text')
+  previewContent?: string | null
+
+  @Column('timestamptz')
+  seenAt?: Date | null
+
+  @Column('boolean')
+  isInLibrary!: boolean
 }

packages/api/src/entity/subscription.ts

@@ -70,4 +70,10 @@ export class Subscription {
 
   @Column('timestamp', { nullable: true })
   scheduledAt?: Date | null
+
+  @Column('boolean')
+  isPublic?: boolean | null
+
+  @Column('boolean')
+  isFetchingContent?: boolean | null
 }

packages/api/src/generated/graphql.ts

@@ -2678,6 +2678,8 @@ export enum SubscribeErrorCode {
 }
 
 export type SubscribeInput = {
+  isFetchingContent?: InputMaybe<Scalars['Boolean']>;
+  isPublic?: InputMaybe<Scalars['Boolean']>;
   subscriptionType?: InputMaybe<SubscriptionType>;
   url: Scalars['String'];
 };
@@ -3038,6 +3040,8 @@ export enum UpdateSubscriptionErrorCode {
 export type UpdateSubscriptionInput = {
   description?: InputMaybe<Scalars['String']>;
   id: Scalars['ID'];
+  isFetchingContent?: InputMaybe<Scalars['Boolean']>;
+  isPublic?: InputMaybe<Scalars['Boolean']>;
   lastFetchedAt?: InputMaybe<Scalars['Date']>;
   lastFetchedChecksum?: InputMaybe<Scalars['String']>;
   name?: InputMaybe<Scalars['String']>;

packages/api/src/generated/schema.graphql

@@ -2112,6 +2112,8 @@ enum SubscribeErrorCode {
 }
 
 input SubscribeInput {
+  isFetchingContent: Boolean
+  isPublic: Boolean
   subscriptionType: SubscriptionType
   url: String!
 }
@@ -2443,6 +2445,8 @@ enum UpdateSubscriptionErrorCode {
 input UpdateSubscriptionInput {
   description: String
   id: ID!
+  isFetchingContent: Boolean
+  isPublic: Boolean
   lastFetchedAt: Date
   lastFetchedChecksum: String
   name: String

packages/api/src/resolvers/subscriptions/index.ts

@@ -213,6 +213,7 @@ export const subscribeResolver = authorized<
         scheduledDates: [new Date()], // fetch immediately
         fetchedDates: [updatedSubscription.lastFetchedAt || null],
         checksums: [updatedSubscription.lastFetchedChecksum || null],
+        isFetchingContents: [!!updatedSubscription.isFetchingContent],
       })
 
       return {
@@ -228,7 +229,7 @@ export const subscribeResolver = authorized<
     // limit number of rss subscriptions to 150
     const results = (await getRepository(Subscription).query(
       `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) 
-          select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions 
+          select $1, $2, $3, $4, $5, $6, $8 from omnivore.subscriptions 
           where user_id = $5 and type = 'RSS' and status = 'ACTIVE' 
           having count(*) < $7 
           returning *;`,
@@ -240,6 +241,7 @@ export const subscribeResolver = authorized<
         uid,
         feed.image?.url || null,
         MAX_RSS_SUBSCRIPTIONS,
+        !!input.isFetchingContent,
       ]
     )) as Subscription[]
 
@@ -259,6 +261,7 @@ export const subscribeResolver = authorized<
       scheduledDates: [new Date()], // fetch immediately
       fetchedDates: [null],
       checksums: [null],
+      isFetchingContents: [!!newSubscription.isFetchingContent],
     })
 
     return {

packages/api/src/routers/svc/rss_feed.ts

@@ -34,7 +34,8 @@ export function rssFeedRouter() {
           ARRAY_AGG(user_id) AS "userIds",
           ARRAY_AGG(last_fetched_at) AS "fetchedDates",
           ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates",
-          ARRAY_AGG(last_fetched_checksum) AS checksums
+          ARRAY_AGG(last_fetched_checksum) AS checksums,
+          ARRAY_AGG(coalesce(is_fetching_content, false)) AS "isFetchingContents",
         FROM
           omnivore.subscriptions
         WHERE

packages/api/src/schema.ts

@@ -2554,6 +2554,8 @@ const schema = gql`
   input SubscribeInput {
     url: String!
     subscriptionType: SubscriptionType
+    isPublic: Boolean
+    isFetchingContent: Boolean
   }
 
   input UpdateSubscriptionInput {
@@ -2564,6 +2566,8 @@ const schema = gql`
     lastFetchedChecksum: String
     status: SubscriptionStatus
     scheduledAt: Date
+    isPublic: Boolean
+    isFetchingContent: Boolean
   }
 
   union UpdateSubscriptionResult =

packages/api/src/utils/createTask.ts

@@ -615,6 +615,7 @@ export interface RssSubscriptionGroup {
   fetchedDates: (Date | null)[]
   scheduledDates: Date[]
   checksums: (string | null)[]
+  isFetchingContents: boolean[]
 }
 
 export const enqueueRssFeedFetch = async (
@@ -632,6 +633,7 @@ export const enqueueRssFeedFetch = async (
       timestamp.getTime()
     ), // unix timestamp in milliseconds
     userIds: subscriptionGroup.userIds,
+    isFetchingContents: subscriptionGroup.isFetchingContents,
   }
 
   // If there is no Google Cloud Project Id exposed, it means that we are in local environment

packages/rss-handler/src/index.ts

@@ -14,6 +14,7 @@ interface RssFeedRequest {
   scheduledTimestamps: number[] // unix timestamp in milliseconds
   lastFetchedChecksums: string[]
   userIds: string[]
+  isFetchingContents: boolean[]
 }
 
 // link can be a string or an object
@@ -26,7 +27,8 @@ function isRssFeedRequest(body: any): body is RssFeedRequest {
     'lastFetchedTimestamps' in body &&
     'scheduledTimestamps' in body &&
     'userIds' in body &&
-    'lastFetchedChecksums' in body
+    'lastFetchedChecksums' in body &&
+    'isFetchingContents' in body
   )
 }