Merge pull request #3139 from omnivore-app/change-folder-to-state-1

packages/api/src/generated/graphql.ts

@@ -1867,6 +1867,7 @@ export type QueryTypeaheadSearchArgs = {
 export type QueryUpdatesSinceArgs = {
   after?: InputMaybe<Scalars['String']>;
   first?: InputMaybe<Scalars['Int']>;
+  folder?: InputMaybe<Scalars['String']>;
   since: Scalars['Date'];
   sort?: InputMaybe<SortParams>;
 };
@@ -2466,6 +2467,7 @@ export type SetIntegrationInput = {
   importItemState?: InputMaybe<ImportItemState>;
   name: Scalars['String'];
   syncedAt?: InputMaybe<Scalars['Date']>;
+  taskName?: InputMaybe<Scalars['String']>;
   token: Scalars['String'];
   type?: InputMaybe<IntegrationType>;
 };

packages/api/src/generated/schema.graphql

@@ -1368,7 +1368,7 @@ type Query {
   sendInstallInstructions: SendInstallInstructionsResult!
   subscriptions(sort: SortParams, type: SubscriptionType): SubscriptionsResult!
   typeaheadSearch(first: Int, query: String!): TypeaheadSearchResult!
-  updatesSince(after: String, first: Int, since: Date!, sort: SortParams): UpdatesSinceResult!
+  updatesSince(after: String, first: Int, folder: String, since: Date!, sort: SortParams): UpdatesSinceResult!
   user(userId: ID, username: String): UserResult!
   users: UsersResult!
   validateUsername(username: String!): Boolean!
@@ -1909,6 +1909,7 @@ input SetIntegrationInput {
   importItemState: ImportItemState
   name: String!
   syncedAt: Date
+  taskName: String
   token: String!
   type: IntegrationType
 }

packages/api/src/repository/label.ts

@@ -32,7 +32,10 @@ const convertToLabel = (label: CreateLabelInput, userId: string) => {
   return {
     user: { id: userId },
     name: label.name,
-    color: label.color || generateRandomColor(), // assign a random color if not provided
+    color:
+      label.color ||
+      getInternalLabelWithColor(label.name)?.color ||
+      generateRandomColor(), // assign a random color if not provided
     description: label.description,
     internal: isLabelInternal(label.name),
   }

packages/api/src/resolvers/article/index.ts

@@ -5,8 +5,9 @@
 /* eslint-disable @typescript-eslint/no-floating-promises */
 import { Readability } from '@omnivore/readability'
 import graphqlFields from 'graphql-fields'
+import { IsNull } from 'typeorm'
 import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
-import { LibraryItem } from '../../entity/library_item'
+import { LibraryItem, LibraryItemState } from '../../entity/library_item'
 import { env } from '../../env'
 import {
   ArticleError,
@@ -390,7 +391,10 @@ export const getArticleResolver = authorized<
     const libraryItem = await authTrx((tx) =>
       tx.withRepository(libraryItemRepository).findOne({
         select: selectColumns,
-        where,
+        where: {
+          ...where,
+          deletedAt: IsNull(),
+        },
         relations: {
           labels: true,
           highlights: {
@@ -406,7 +410,7 @@ export const getArticleResolver = authorized<
       })
     )
 
-    if (!libraryItem || libraryItem.folder === InFilter.TRASH) {
+    if (!libraryItem) {
       return { errorCodes: [ArticleErrorCode.NotFound] }
     }
 
@@ -528,8 +532,8 @@ export const setBookmarkArticleResolver = authorized<
   const deletedLibraryItem = await updateLibraryItem(
     articleID,
     {
-      folder: InFilter.TRASH,
-      savedAt: new Date(),
+      state: LibraryItemState.Deleted,
+      deletedAt: new Date(),
     },
     uid,
     pubsub
@@ -737,7 +741,7 @@ export const updatesSinceResolver = authorized<
   UpdatesSinceSuccess,
   UpdatesSinceError,
   QueryUpdatesSinceArgs
->(async (_obj, { since, first, after, sort: sortParams }, { uid }) => {
+>(async (_obj, { since, first, after, sort: sortParams, folder }, { uid }) => {
   const sort = sortParamsToSort(sortParams)
 
   const startCursor = after || ''
@@ -755,7 +759,7 @@ export const updatesSinceResolver = authorized<
       includeDeleted: true,
       dateFilters: [{ field: 'updatedAt', startDate }],
       sort,
-      inFilter: InFilter.ALL,
+      inFilter: (folder as InFilter) || InFilter.ALL,
     },
     uid
   )
@@ -867,7 +871,7 @@ export const setFavoriteArticleResolver = authorized<
 })
 
 const getUpdateReason = (libraryItem: LibraryItem, since: Date) => {
-  if (libraryItem.folder === InFilter.TRASH) {
+  if (libraryItem.deletedAt) {
     return UpdateReason.Deleted
   }
   if (libraryItem.createdAt >= since) {

packages/api/src/resolvers/integrations/index.ts

@@ -216,8 +216,6 @@ export const importFromIntegrationResolver = authorized<
   ImportFromIntegrationError,
   MutationImportFromIntegrationArgs
 >(async (_, { integrationId }, { claims: { uid }, log }) => {
-  log.info('importFromIntegrationResolver')
-
   try {
     const integration = await findIntegration({ id: integrationId }, uid)
 

packages/api/src/resolvers/links/index.ts

@@ -1,3 +1,4 @@
+import { LibraryItemState } from '../../entity/library_item'
 import { env } from '../../env'
 import {
   ArchiveLinkError,
@@ -8,7 +9,6 @@ import {
 import { updateLibraryItem } from '../../services/library_item'
 import { analytics } from '../../utils/analytics'
 import { authorized } from '../../utils/helpers'
-import { InFilter } from '../../utils/search'
 
 // export const updateLinkShareInfoResolver = authorized<
 //   UpdateLinkShareInfoSuccess,
@@ -54,9 +54,20 @@ export const setLinkArchivedResolver = authorized<
   ArchiveLinkError,
   MutationSetLinkArchivedArgs
 >(async (_obj, args, { uid }) => {
+  let state = LibraryItemState.Archived
+  let archivedAt: Date | null = new Date()
+  let event = 'link_archived'
+
+  const isUnarchive = !args.input.archived
+  if (isUnarchive) {
+    state = LibraryItemState.Succeeded
+    archivedAt = null
+    event = 'link_unarchived'
+  }
+
   analytics.track({
     userId: uid,
-    event: args.input.archived ? 'link_archived' : 'link_unarchived',
+    event,
     properties: {
       env: env.server.apiEnv,
     },
@@ -66,8 +77,8 @@ export const setLinkArchivedResolver = authorized<
     await updateLibraryItem(
       args.input.linkId,
       {
-        savedAt: new Date(),
-        folder: args.input.archived ? InFilter.ARCHIVE : InFilter.INBOX,
+        state,
+        archivedAt,
       },
       uid
     )
@@ -80,6 +91,6 @@ export const setLinkArchivedResolver = authorized<
 
   return {
     linkId: args.input.linkId,
-    message: 'Link Archived',
+    message: event,
   }
 })

packages/api/src/resolvers/update/index.ts

@@ -1,3 +1,4 @@
+import { LibraryItemState } from '../../entity/library_item'
 import {
   MutationUpdatePageArgs,
   UpdatePageError,
@@ -20,6 +21,9 @@ export const updatePageResolver = authorized<
       savedAt: input.savedAt ? new Date(input.savedAt) : undefined,
       publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined,
       thumbnail: input.previewImage ?? undefined,
+      state: input.state
+        ? (input.state as unknown as LibraryItemState)
+        : undefined,
     },
     uid
   )

packages/api/src/routers/svc/following.ts

@@ -1,5 +1,9 @@
 /* eslint-disable @typescript-eslint/no-misused-promises */
 import express from 'express'
+import {
+  findOrCreateLabels,
+  saveLabelsInLibraryItem,
+} from '../../services/labels'
 import { saveFeedItemInFollowing } from '../../services/library_item'
 import { logger } from '../../utils/logger'
 
@@ -19,6 +23,7 @@ export interface SaveFollowingItemRequest {
   previewContentType?: string
   publishedAt?: Date
   savedAt?: Date
+  thumbnail?: string
 }
 
 function isSaveFollowingItemRequest(
@@ -49,16 +54,40 @@ export function followingServiceRouter() {
       return res.status(400).send('INVALID_REQUEST_BODY')
     }
 
-    if (req.body.addedToFollowingFrom === 'feed') {
-      logger.info('saving feed item')
+    if (
+      req.body.addedToFollowingFrom === 'feed' &&
+      req.body.userIds.length > 0
+    ) {
+      const userId = req.body.userIds[0]
+      logger.info('saving feed item', userId)
 
-      const result = await saveFeedItemInFollowing(req.body)
+      const result = await saveFeedItemInFollowing(req.body, userId)
       if (result.identifiers.length === 0) {
         logger.error('error saving feed item in following')
         return res.status(500).send('ERROR_SAVING_FEED_ITEM')
       }
 
       logger.info('feed item saved in following')
+
+      // add RSS label to the item
+      const labels = await findOrCreateLabels(
+        [
+          {
+            name: 'RSS',
+          },
+        ],
+        userId
+      )
+      await saveLabelsInLibraryItem(
+        labels,
+        result.identifiers[0].id,
+        userId,
+        undefined,
+        true
+      )
+
+      logger.info('RSS label added to the item')
+
       return res.sendStatus(200)
     }
 

packages/api/src/schema.ts

@@ -1997,6 +1997,7 @@ const schema = gql`
     enabled: Boolean!
     syncedAt: Date
     importItemState: ImportItemState
+    taskName: String
   }
 
   union IntegrationsResult = IntegrationsSuccess | IntegrationsError
@@ -2826,6 +2827,7 @@ const schema = gql`
       first: Int
       since: Date!
       sort: SortParams
+      folder: String
     ): UpdatesSinceResult!
     integrations: IntegrationsResult!
     recentSearches: RecentSearchesResult!

packages/api/src/services/labels.ts

@@ -69,7 +69,8 @@ export const saveLabelsInLibraryItem = async (
   labels: Label[],
   libraryItemId: string,
   userId: string,
-  pubsub = createPubSubClient()
+  pubsub = createPubSubClient(),
+  skipPubSub = false
 ) => {
   await authTrx(
     async (tx) => {
@@ -92,6 +93,10 @@ export const saveLabelsInLibraryItem = async (
     userId
   )
 
+  if (skipPubSub) {
+    return
+  }
+
   // create pubsub event
   await pubsub.entityCreated<AddLabelsToLibraryItemEvent>(
     EntityType.LABEL,
@@ -104,7 +109,8 @@ export const addLabelsToLibraryItem = async (
   labels: Label[],
   libraryItemId: string,
   userId: string,
-  pubsub = createPubSubClient()
+  pubsub = createPubSubClient(),
+  skipPubSub = false
 ) => {
   await authTrx(
     async (tx) => {
@@ -128,6 +134,10 @@ export const addLabelsToLibraryItem = async (
     userId
   )
 
+  if (skipPubSub) {
+    return
+  }
+
   // create pubsub event
   await pubsub.entityCreated<AddLabelsToLibraryItemEvent>(
     EntityType.LABEL,

packages/api/src/services/library_item.ts

@@ -9,8 +9,8 @@ import { createPubSubClient, EntityType } from '../pubsub'
 import { authTrx, getColumns } from '../repository'
 import { libraryItemRepository } from '../repository/library_item'
 import { SaveFollowingItemRequest } from '../routers/svc/following'
-import { SetClaimsRole } from '../utils/dictionary'
 import { generateSlug, wordsCount } from '../utils/helpers'
+import { createThumbnailUrl } from '../utils/imageproxy'
 import {
   DateFilter,
   FieldFilter,
@@ -105,9 +105,24 @@ const buildWhereClause = (
   }
 
   if (args.inFilter !== InFilter.ALL) {
-    queryBuilder.andWhere('library_item.folder = :folder', {
-      folder: args.inFilter,
-    })
+    switch (args.inFilter) {
+      case InFilter.INBOX:
+        queryBuilder.andWhere('library_item.archived_at IS NULL')
+        break
+      case InFilter.ARCHIVE:
+        queryBuilder.andWhere('library_item.archived_at IS NOT NULL')
+        break
+      case InFilter.TRASH:
+        // return only deleted pages within 14 days
+        queryBuilder.andWhere(
+          "library_item.deleted_at >= now() - interval '14 days'"
+        )
+        break
+      default:
+        queryBuilder.andWhere('library_item.folder = :folder', {
+          folder: args.inFilter,
+        })
+    }
   }
 
   if (args.readFilter !== ReadFilter.ALL) {
@@ -227,13 +242,13 @@ const buildWhereClause = (
   }
 
   if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) {
-    queryBuilder.andWhere("library_item.folder <> 'trash'")
+    queryBuilder.andWhere("library_item.state <> 'DELETED'")
   }
 
   if (args.noFilters) {
     args.noFilters.forEach((filter) => {
       queryBuilder.andWhere(
-        `library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL`
+        `(library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL)`
       )
     })
   }
@@ -365,6 +380,8 @@ export const restoreLibraryItem = async (
     {
       state: LibraryItemState.Succeeded,
       savedAt: new Date(),
+      archivedAt: null,
+      deletedAt: null,
     },
     userId,
     pubsub
@@ -380,6 +397,21 @@ export const updateLibraryItem = async (
   const updatedLibraryItem = await authTrx(
     async (tx) => {
       const itemRepo = tx.withRepository(libraryItemRepository)
+
+      // reset deletedAt and archivedAt
+      switch (libraryItem.state) {
+        case LibraryItemState.Archived:
+          libraryItem.archivedAt = new Date()
+          break
+        case LibraryItemState.Deleted:
+          libraryItem.deletedAt = new Date()
+          break
+        case LibraryItemState.Processing:
+        case LibraryItemState.Succeeded:
+          libraryItem.archivedAt = null
+          libraryItem.deletedAt = null
+          break
+      }
       await itemRepo.update(id, libraryItem)
 
       return itemRepo.findOneByOrFail({ id })
@@ -519,31 +551,35 @@ export const createLibraryItem = async (
   return newLibraryItem
 }
 
-export const saveFeedItemInFollowing = (input: SaveFollowingItemRequest) => {
+export const saveFeedItemInFollowing = (
+  input: SaveFollowingItemRequest,
+  userId: string
+) => {
+  const thumbnail = input.thumbnail && createThumbnailUrl(input.thumbnail)
+
   return authTrx(
     async (tx) => {
-      const libraryItems: QueryDeepPartialEntity<LibraryItem>[] =
-        input.userIds.map((userId) => ({
-          ...input,
-          user: { id: userId },
-          originalUrl: input.url,
-          subscription: input.addedToFollowingBy,
-          folder: InFilter.FOLLOWING,
-          slug: generateSlug(input.title),
-        }))
+      const itemToSave: QueryDeepPartialEntity<LibraryItem> = {
+        ...input,
+        user: { id: userId },
+        originalUrl: input.url,
+        subscription: input.addedToFollowingBy,
+        folder: InFilter.FOLLOWING,
+        slug: generateSlug(input.title),
+        thumbnail,
+      }
 
       return tx
         .getRepository(LibraryItem)
         .createQueryBuilder()
         .insert()
-        .values(libraryItems)
+        .values(itemToSave)
         .orIgnore() // ignore if the item already exists
         .returning('*')
         .execute()
     },
     undefined,
-    undefined,
-    SetClaimsRole.ADMIN
+    userId
   )
 }
 
@@ -600,14 +636,14 @@ export const updateLibraryItems = async (
   switch (action) {
     case BulkActionType.Archive:
       values = {
-        folder: InFilter.ARCHIVE,
-        savedAt: new Date(),
+        archivedAt: new Date(),
+        state: LibraryItemState.Archived,
       }
       break
     case BulkActionType.Delete:
       values = {
-        savedAt: new Date(),
-        folder: InFilter.TRASH,
+        state: LibraryItemState.Deleted,
+        deletedAt: new Date(),
       }
       break
     case BulkActionType.AddLabels:

packages/api/src/services/save_page.ts

@@ -261,6 +261,9 @@ export const parsedContentToLibraryItem = ({
     uploadFileId: uploadFileId || undefined,
     readingProgressTopPercent: 0,
     readingProgressHighestReadAnchor: 0,
+    state: state
+      ? (state as unknown as LibraryItemState)
+      : LibraryItemState.Succeeded,
     createdAt: validatedDate(saveTime),
     savedAt: validatedDate(saveTime),
     siteName: parsedContent?.siteName,
@@ -269,7 +272,8 @@ export const parsedContentToLibraryItem = ({
     wordCount: wordsCount(parsedContent?.textContent || ''),
     contentReader: contentReaderForLibraryItem(itemType, uploadFileId),
     subscription: rssFeedUrl,
-    folder: state === ArticleSavingRequestStatus.Archived ? 'archive' : 'inbox',
-    state: LibraryItemState.Succeeded,
+    folder: 'inbox',
+    archivedAt:
+      state === ArticleSavingRequestStatus.Archived ? new Date() : null,
   }
 }

packages/api/src/utils/helpers.ts

@@ -240,7 +240,7 @@ export const libraryItemToArticle = (item: LibraryItem): Article => ({
   state: item.state as unknown as ArticleSavingRequestStatus,
   content: item.readableContent,
   hash: item.textContentHash || '',
-  isArchived: item.folder === InFilter.ARCHIVE,
+  isArchived: !!item.archivedAt,
   recommendations: item.recommendations?.map(
     recommandationDataToRecommendation
   ),
@@ -259,7 +259,7 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({
   url: item.originalUrl,
   state: item.state as unknown as ArticleSavingRequestStatus,
   content: item.readableContent,
-  isArchived: item.folder === InFilter.ARCHIVE,
+  isArchived: !!item.archivedAt,
   pageType: item.itemType as unknown as PageType,
   readingProgressPercent: item.readingProgressBottomPercent,
   contentReader: item.contentReader as unknown as ContentReader,

packages/api/src/utils/imageproxy.ts

@@ -27,3 +27,6 @@ export function createImageProxyUrl(
 
   return `${env.imageProxy.url}/${width}x${height},s${signature}/${url}`
 }
+
+export const createThumbnailUrl = (url: string): string =>
+  createImageProxyUrl(url, 320, 320)

packages/api/test/resolvers/article.test.ts

@@ -410,7 +410,8 @@ describe('Article API', () => {
             title,
             user: { id: user.id },
             originalUrl: url,
-            folder: 'archive',
+            archivedAt: new Date(),
+            state: LibraryItemState.Archived,
           },
           user.id
         )
@@ -608,7 +609,7 @@ describe('Article API', () => {
         ).expect(200)
 
         const savedItem = await findLibraryItemByUrl(url, user.id)
-        expect(savedItem?.folder).to.eql('archive')
+        expect(savedItem?.archivedAt).to.not.be.null
         expect(savedItem?.labels?.map((l) => l.name)).to.eql(labels)
       })
     })
@@ -691,7 +692,7 @@ describe('Article API', () => {
         200
       )
       const item = await findLibraryItemById(itemId, user.id)
-      expect(item?.folder).to.eql('trash')
+      expect(item?.state).to.eql(LibraryItemState.Deleted)
     })
   })
 
@@ -1030,7 +1031,8 @@ describe('Article API', () => {
               readableContent: '<p>test 1</p>',
               slug: 'test slug 1',
               originalUrl: `${url}/test1`,
-              folder: 'archive',
+              archivedAt: new Date(),
+              state: LibraryItemState.Archived,
             },
             {
               user,
@@ -1038,7 +1040,8 @@ describe('Article API', () => {
               readableContent: '<p>test 2</p>',
               slug: 'test slug 2',
               originalUrl: `${url}/test2`,
-              folder: 'archive',
+              archivedAt: new Date(),
+              state: LibraryItemState.Archived,
             },
             {
               user,
@@ -1150,12 +1153,12 @@ describe('Article API', () => {
       })
     })
 
-    context("when in:inbox label:test' is in the query", () => {
+    context('when in:inbox no:subscription label:test is in the query', () => {
       let items: LibraryItem[] = []
       let label: Label
 
       before(async () => {
-        keyword = 'in:inbox label:test'
+        keyword = 'in:inbox no:subscription label:test'
         // Create some test items
         label = await createLabel('test', '', user.id)
         items = await createLibraryItems(
@@ -1167,19 +1170,29 @@ describe('Article API', () => {
               slug: 'test slug 1',
               originalUrl: `${url}/test1`,
             },
+            {
+              user,
+              title: 'test title 2',
+              readableContent: '<p>test 2</p>',
+              slug: 'test slug 2',
+              originalUrl: `${url}/test2`,
+              subscription: 'test subscription',
+            },
             {
               user,
               title: 'test title 3',
               readableContent: '<p>test 3</p>',
               slug: 'test slug 3',
               originalUrl: `${url}/test3`,
-              folder: 'archive',
+              archivedAt: new Date(),
+              state: LibraryItemState.Archived,
             },
           ],
           user.id
         )
         await saveLabelsInLibraryItem([label], items[0].id, user.id)
         await saveLabelsInLibraryItem([label], items[1].id, user.id)
+        await saveLabelsInLibraryItem([label], items[2].id, user.id)
       })
 
       after(async () => {
@@ -1263,7 +1276,7 @@ describe('Article API', () => {
               slug: 'test slug 1',
               originalUrl: `${url}/test1`,
               itemType: PageType.File,
-              folder: 'archive',
+              archivedAt: new Date(),
             },
             {
               user,
@@ -1271,7 +1284,7 @@ describe('Article API', () => {
               readableContent: '<p>test 2</p>',
               slug: 'test slug 2',
               originalUrl: `${url}/test2`,
-              folder: 'archive',
+              archivedAt: new Date(),
               readingProgressBottomPercent: 100,
             },
             {
@@ -1313,7 +1326,7 @@ describe('Article API', () => {
               slug: 'test slug 1',
               originalUrl: `${url}/test1`,
               subscription: 'feed',
-              folder: 'archive',
+              archivedAt: new Date(),
             },
             {
               user,
@@ -1329,7 +1342,7 @@ describe('Article API', () => {
               readableContent: '<p>test 3</p>',
               slug: 'test slug 3',
               originalUrl: `${url}/test3`,
-              folder: 'archive',
+              archivedAt: new Date(),
             },
           ],
           user.id
@@ -1362,7 +1375,7 @@ describe('Article API', () => {
               readableContent: '<p>test 1</p>',
               slug: 'test slug 1',
               originalUrl: `${url}/test1`,
-              folder: 'trash',
+              deletedAt: new Date(),
             },
             {
               user,
@@ -1371,7 +1384,7 @@ describe('Article API', () => {
               slug: 'test slug 2',
               originalUrl: `${url}/test2`,
               readingProgressBottomPercent: 100,
-              folder: 'trash',
+              deletedAt: new Date(),
             },
             {
               user,
@@ -1733,7 +1746,7 @@ describe('Article API', () => {
       for (let i = 0; i < 3; i++) {
         await updateLibraryItem(
           items[i].id,
-          { folder: 'trash', savedAt: new Date() },
+          { state: LibraryItemState.Deleted, deletedAt: new Date() },
           user.id
         )
         deletedItems.push(items[i])

packages/db/migrations/0148.do.update_folder_in_library_item.sql

@@ -0,0 +1,9 @@
+-- Type: DO
+-- Name: update_folder_in_library_item
+-- Description: Update folder column in library_item table
+
+BEGIN;
+
+UPDATE omnivore.library_item SET folder = 'inbox' WHERE folder = 'archive' OR folder = 'trash';
+
+COMMIT;

packages/db/migrations/0148.undo.update_folder_in_library_item.sql

@@ -0,0 +1,10 @@
+-- Type: UNDO
+-- Name: update_folder_in_library_item
+-- Description: Update folder column in library_item table
+
+BEGIN;
+
+UPDATE omnivore.library_item SET folder = 'archive' WHERE archived_at IS NOT NULL;
+UPDATE omnivore.library_item SET folder = 'trash' WHERE deleted_at IS NOT NULL;
+
+COMMIT;

packages/integration-handler/src/index.ts

@@ -251,33 +251,33 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction(
             total: offset,
             size: retrievedData.length,
           })
-
-          console.log('updating integration...', {
-            userId,
-            integrationId: req.body.integrationId,
-            syncedAt,
-          })
-          // update the integration's syncedAt and remove taskName
-          const result = await updateIntegration(
-            REST_BACKEND_ENDPOINT,
-            req.body.integrationId,
-            new Date(syncedAt),
-            req.body.integrationName,
-            claims.token,
-            token,
-            'IMPORT',
-            null
-          )
-          if (!result) {
-            console.error('failed to update integration', {
-              userId,
-              integrationId: req.body.integrationId,
-            })
-            return res.status(400).send('Failed to update integration')
-          }
         } while (retrievedData.length > 0 && offset < 20000) // limit to 20k pages
       }
 
+      console.log('updating integration...', {
+        userId,
+        integrationId: req.body.integrationId,
+        syncedAt,
+      })
+      // update the integration's syncedAt and remove taskName
+      const result = await updateIntegration(
+        REST_BACKEND_ENDPOINT,
+        req.body.integrationId,
+        new Date(syncedAt),
+        req.body.integrationName,
+        claims.token,
+        token,
+        'IMPORT',
+        null
+      )
+      if (!result) {
+        console.error('failed to update integration', {
+          userId,
+          integrationId: req.body.integrationId,
+        })
+        return res.status(500).send('Failed to update integration')
+      }
+
       console.log('done')
     } catch (err) {
       console.error('import pages from integration failed', {

packages/rss-handler/src/index.ts

@@ -19,6 +19,35 @@ interface RssFeedRequest {
 
 // link can be a string or an object
 type RssFeedItemLink = string | { $: { rel?: string; href: string } }
+type RssFeed = Parser.Output<{
+  published?: string
+  updated?: string
+  created?: string
+  link?: RssFeedItemLink
+  links?: RssFeedItemLink[]
+}> & {
+  lastBuildDate?: string
+  'syn:updatePeriod'?: string
+  'syn:updateFrequency'?: string
+  'sy:updatePeriod'?: string
+  'sy:updateFrequency'?: string
+}
+type RssFeedItemMedia = {
+  $: { url: string; width?: string; height?: string; medium?: string }
+}
+type RssFeedItem = Item & {
+  'media:thumbnail'?: RssFeedItemMedia
+  'media:content'?: RssFeedItemMedia[]
+}
+
+const getThumbnail = (item: RssFeedItem) => {
+  if (item['media:thumbnail']) {
+    return item['media:thumbnail'].$.url
+  }
+
+  return item['media:content']?.find((media) => media.$.medium === 'image')?.$
+    .url
+}
 
 function isRssFeedRequest(body: any): body is RssFeedRequest {
   return (
@@ -125,7 +154,7 @@ const sendUpdateSubscriptionMutation = async (
 const createTask = async (
   userId: string,
   feedUrl: string,
-  item: Item,
+  item: RssFeedItem,
   autoAddToLibrary: boolean
 ) => {
   if (autoAddToLibrary) {
@@ -138,7 +167,7 @@ const createTask = async (
 const createSavingItemTask = async (
   userId: string,
   feedUrl: string,
-  item: Item
+  item: RssFeedItem
 ) => {
   const input = {
     userId,
@@ -167,7 +196,7 @@ const createSavingItemTask = async (
 const createFollowingTask = async (
   userId: string,
   feedUrl: string,
-  item: Item
+  item: RssFeedItem
 ) => {
   const input = {
     userIds: [userId],
@@ -181,6 +210,7 @@ const createFollowingTask = async (
     savedAt: item.isoDate,
     publishedAt: item.isoDate,
     previewContentType: 'text/html', // TODO: get content type from feed
+    thumbnail: getThumbnail(item),
   }
 
   try {
@@ -217,6 +247,8 @@ const parser = new Parser({
       'published',
       'updated',
       'created',
+      ['media:content', 'media:content', { keepArray: true }],
+      ['media:thumbnail'],
     ],
     feed: [
       'lastBuildDate',
@@ -228,9 +260,9 @@ const parser = new Parser({
   },
 })
 
-const getUpdateFrequency = (feed: any) => {
-  const updateFrequency = (feed['syn:updateFrequency'] ||
-    feed['sy:updateFrequency']) as string | undefined
+const getUpdateFrequency = (feed: RssFeed) => {
+  const updateFrequency =
+    feed['syn:updateFrequency'] || feed['sy:updateFrequency']
 
   if (!updateFrequency) {
     return 1
@@ -244,10 +276,8 @@ const getUpdateFrequency = (feed: any) => {
   return frequency
 }
 
-const getUpdatePeriodInHours = (feed: any) => {
-  const updatePeriod = (feed['syn:updatePeriod'] || feed['sy:updatePeriod']) as
-    | string
-    | undefined
+const getUpdatePeriodInHours = (feed: RssFeed) => {
+  const updatePeriod = feed['syn:updatePeriod'] || feed['sy:updatePeriod']
 
   switch (updatePeriod) {
     case 'hourly':
@@ -301,19 +331,7 @@ const processSubscription = async (
   scheduledAt: number,
   lastFetchedChecksum: string,
   autoAddToLibrary: boolean,
-  feed: {
-    lastBuildDate: any
-    'syn:updatePeriod': any
-    'syn:updateFrequency': any
-    'sy:updatePeriod': any
-    'sy:updateFrequency': any
-  } & Parser.Output<{
-    published: any
-    updated: any
-    created: any
-    link: any
-    links: any[]
-  }>
+  feed: RssFeed
 ) => {
   let lastItemFetchedAt: Date | null = null
   let lastValidItem: Item | null = null
@@ -327,7 +345,7 @@ const processSubscription = async (
   // fetch feed
   let itemCount = 0
 
-  const feedLastBuildDate = feed.lastBuildDate as string | undefined
+  const feedLastBuildDate = feed.lastBuildDate
   console.log('Feed last build date', feedLastBuildDate)
   if (
     feedLastBuildDate &&
@@ -341,10 +359,7 @@ const processSubscription = async (
   for (const item of feed.items) {
     // use published or updated if isoDate is not available for atom feeds
     item.isoDate =
-      item.isoDate ||
-      (item.published as string) ||
-      (item.updated as string) ||
-      (item.created as string)
+      item.isoDate || item.published || item.updated || item.created
     console.log('Processing feed item', item.links, item.isoDate)
 
     if (!item.links || item.links.length === 0) {
@@ -352,7 +367,7 @@ const processSubscription = async (
       continue
     }
 
-    item.link = getLink(item.links as RssFeedItemLink[])
+    item.link = getLink(item.links)
     if (!item.link) {
       console.log('Invalid feed item links', item.links)
       continue

packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx

@@ -242,9 +242,9 @@ function Subscriptions(
       {!collapsed ? (
         <>
           <FilterButton filterTerm="in:inbox has:subscriptions" text="All" {...props} />
-          <FilterButton filterTerm={`label:RSS`} text="Feeds" {...props} />
+          <FilterButton filterTerm={`in:inbox label:RSS`} text="Feeds" {...props} />
           <FilterButton
-            filterTerm={`label:Newsletter`}
+            filterTerm={`in:inbox label:Newsletter`}
             text="Newsletters"
             {...props}
           />