diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 4e2d86713..fe6a395b8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1867,6 +1867,7 @@ export type QueryTypeaheadSearchArgs = { export type QueryUpdatesSinceArgs = { after?: InputMaybe; first?: InputMaybe; + folder?: InputMaybe; since: Scalars['Date']; sort?: InputMaybe; }; @@ -2466,6 +2467,7 @@ export type SetIntegrationInput = { importItemState?: InputMaybe; name: Scalars['String']; syncedAt?: InputMaybe; + taskName?: InputMaybe; token: Scalars['String']; type?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 9eeea6944..2903d85fb 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1368,7 +1368,7 @@ type Query { sendInstallInstructions: SendInstallInstructionsResult! subscriptions(sort: SortParams, type: SubscriptionType): SubscriptionsResult! typeaheadSearch(first: Int, query: String!): TypeaheadSearchResult! - updatesSince(after: String, first: Int, since: Date!, sort: SortParams): UpdatesSinceResult! + updatesSince(after: String, first: Int, folder: String, since: Date!, sort: SortParams): UpdatesSinceResult! user(userId: ID, username: String): UserResult! users: UsersResult! validateUsername(username: String!): Boolean! @@ -1909,6 +1909,7 @@ input SetIntegrationInput { importItemState: ImportItemState name: String! syncedAt: Date + taskName: String token: String! type: IntegrationType } diff --git a/packages/api/src/repository/label.ts b/packages/api/src/repository/label.ts index d21708961..093a34bd9 100644 --- a/packages/api/src/repository/label.ts +++ b/packages/api/src/repository/label.ts @@ -32,7 +32,10 @@ const convertToLabel = (label: CreateLabelInput, userId: string) => { return { user: { id: userId }, name: label.name, - color: label.color || generateRandomColor(), // assign a random color if not provided + color: + label.color || + getInternalLabelWithColor(label.name)?.color || + generateRandomColor(), // assign a random color if not provided description: label.description, internal: isLabelInternal(label.name), } diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 42b209349..7ac91c69c 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -5,8 +5,9 @@ /* eslint-disable @typescript-eslint/no-floating-promises */ import { Readability } from '@omnivore/readability' import graphqlFields from 'graphql-fields' +import { IsNull } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' -import { LibraryItem } from '../../entity/library_item' +import { LibraryItem, LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { ArticleError, @@ -390,7 +391,10 @@ export const getArticleResolver = authorized< const libraryItem = await authTrx((tx) => tx.withRepository(libraryItemRepository).findOne({ select: selectColumns, - where, + where: { + ...where, + deletedAt: IsNull(), + }, relations: { labels: true, highlights: { @@ -406,7 +410,7 @@ export const getArticleResolver = authorized< }) ) - if (!libraryItem || libraryItem.folder === InFilter.TRASH) { + if (!libraryItem) { return { errorCodes: [ArticleErrorCode.NotFound] } } @@ -528,8 +532,8 @@ export const setBookmarkArticleResolver = authorized< const deletedLibraryItem = await updateLibraryItem( articleID, { - folder: InFilter.TRASH, - savedAt: new Date(), + state: LibraryItemState.Deleted, + deletedAt: new Date(), }, uid, pubsub @@ -737,7 +741,7 @@ export const updatesSinceResolver = authorized< UpdatesSinceSuccess, UpdatesSinceError, QueryUpdatesSinceArgs ->(async (_obj, { since, first, after, sort: sortParams }, { uid }) => { +>(async (_obj, { since, first, after, sort: sortParams, folder }, { uid }) => { const sort = sortParamsToSort(sortParams) const startCursor = after || '' @@ -755,7 +759,7 @@ export const updatesSinceResolver = authorized< includeDeleted: true, dateFilters: [{ field: 'updatedAt', startDate }], sort, - inFilter: InFilter.ALL, + inFilter: (folder as InFilter) || InFilter.ALL, }, uid ) @@ -867,7 +871,7 @@ export const setFavoriteArticleResolver = authorized< }) const getUpdateReason = (libraryItem: LibraryItem, since: Date) => { - if (libraryItem.folder === InFilter.TRASH) { + if (libraryItem.deletedAt) { return UpdateReason.Deleted } if (libraryItem.createdAt >= since) { diff --git a/packages/api/src/resolvers/integrations/index.ts b/packages/api/src/resolvers/integrations/index.ts index dc2f861c1..18667b90d 100644 --- a/packages/api/src/resolvers/integrations/index.ts +++ b/packages/api/src/resolvers/integrations/index.ts @@ -216,8 +216,6 @@ export const importFromIntegrationResolver = authorized< ImportFromIntegrationError, MutationImportFromIntegrationArgs >(async (_, { integrationId }, { claims: { uid }, log }) => { - log.info('importFromIntegrationResolver') - try { const integration = await findIntegration({ id: integrationId }, uid) diff --git a/packages/api/src/resolvers/links/index.ts b/packages/api/src/resolvers/links/index.ts index d16b9a155..8950222d1 100644 --- a/packages/api/src/resolvers/links/index.ts +++ b/packages/api/src/resolvers/links/index.ts @@ -1,3 +1,4 @@ +import { LibraryItemState } from '../../entity/library_item' import { env } from '../../env' import { ArchiveLinkError, @@ -8,7 +9,6 @@ import { import { updateLibraryItem } from '../../services/library_item' import { analytics } from '../../utils/analytics' import { authorized } from '../../utils/helpers' -import { InFilter } from '../../utils/search' // export const updateLinkShareInfoResolver = authorized< // UpdateLinkShareInfoSuccess, @@ -54,9 +54,20 @@ export const setLinkArchivedResolver = authorized< ArchiveLinkError, MutationSetLinkArchivedArgs >(async (_obj, args, { uid }) => { + let state = LibraryItemState.Archived + let archivedAt: Date | null = new Date() + let event = 'link_archived' + + const isUnarchive = !args.input.archived + if (isUnarchive) { + state = LibraryItemState.Succeeded + archivedAt = null + event = 'link_unarchived' + } + analytics.track({ userId: uid, - event: args.input.archived ? 'link_archived' : 'link_unarchived', + event, properties: { env: env.server.apiEnv, }, @@ -66,8 +77,8 @@ export const setLinkArchivedResolver = authorized< await updateLibraryItem( args.input.linkId, { - savedAt: new Date(), - folder: args.input.archived ? InFilter.ARCHIVE : InFilter.INBOX, + state, + archivedAt, }, uid ) @@ -80,6 +91,6 @@ export const setLinkArchivedResolver = authorized< return { linkId: args.input.linkId, - message: 'Link Archived', + message: event, } }) diff --git a/packages/api/src/resolvers/update/index.ts b/packages/api/src/resolvers/update/index.ts index 10eb68d32..255efd2b5 100644 --- a/packages/api/src/resolvers/update/index.ts +++ b/packages/api/src/resolvers/update/index.ts @@ -1,3 +1,4 @@ +import { LibraryItemState } from '../../entity/library_item' import { MutationUpdatePageArgs, UpdatePageError, @@ -20,6 +21,9 @@ export const updatePageResolver = authorized< savedAt: input.savedAt ? new Date(input.savedAt) : undefined, publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, thumbnail: input.previewImage ?? undefined, + state: input.state + ? (input.state as unknown as LibraryItemState) + : undefined, }, uid ) diff --git a/packages/api/src/routers/svc/following.ts b/packages/api/src/routers/svc/following.ts index 68a016c60..ef962f7b4 100644 --- a/packages/api/src/routers/svc/following.ts +++ b/packages/api/src/routers/svc/following.ts @@ -1,5 +1,9 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ import express from 'express' +import { + findOrCreateLabels, + saveLabelsInLibraryItem, +} from '../../services/labels' import { saveFeedItemInFollowing } from '../../services/library_item' import { logger } from '../../utils/logger' @@ -19,6 +23,7 @@ export interface SaveFollowingItemRequest { previewContentType?: string publishedAt?: Date savedAt?: Date + thumbnail?: string } function isSaveFollowingItemRequest( @@ -49,16 +54,40 @@ export function followingServiceRouter() { return res.status(400).send('INVALID_REQUEST_BODY') } - if (req.body.addedToFollowingFrom === 'feed') { - logger.info('saving feed item') + if ( + req.body.addedToFollowingFrom === 'feed' && + req.body.userIds.length > 0 + ) { + const userId = req.body.userIds[0] + logger.info('saving feed item', userId) - const result = await saveFeedItemInFollowing(req.body) + const result = await saveFeedItemInFollowing(req.body, userId) if (result.identifiers.length === 0) { logger.error('error saving feed item in following') return res.status(500).send('ERROR_SAVING_FEED_ITEM') } logger.info('feed item saved in following') + + // add RSS label to the item + const labels = await findOrCreateLabels( + [ + { + name: 'RSS', + }, + ], + userId + ) + await saveLabelsInLibraryItem( + labels, + result.identifiers[0].id, + userId, + undefined, + true + ) + + logger.info('RSS label added to the item') + return res.sendStatus(200) } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e532f1165..1e774fc0b 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1997,6 +1997,7 @@ const schema = gql` enabled: Boolean! syncedAt: Date importItemState: ImportItemState + taskName: String } union IntegrationsResult = IntegrationsSuccess | IntegrationsError @@ -2826,6 +2827,7 @@ const schema = gql` first: Int since: Date! sort: SortParams + folder: String ): UpdatesSinceResult! integrations: IntegrationsResult! recentSearches: RecentSearchesResult! diff --git a/packages/api/src/services/labels.ts b/packages/api/src/services/labels.ts index a33e1e2e6..a015098be 100644 --- a/packages/api/src/services/labels.ts +++ b/packages/api/src/services/labels.ts @@ -69,7 +69,8 @@ export const saveLabelsInLibraryItem = async ( labels: Label[], libraryItemId: string, userId: string, - pubsub = createPubSubClient() + pubsub = createPubSubClient(), + skipPubSub = false ) => { await authTrx( async (tx) => { @@ -92,6 +93,10 @@ export const saveLabelsInLibraryItem = async ( userId ) + if (skipPubSub) { + return + } + // create pubsub event await pubsub.entityCreated( EntityType.LABEL, @@ -104,7 +109,8 @@ export const addLabelsToLibraryItem = async ( labels: Label[], libraryItemId: string, userId: string, - pubsub = createPubSubClient() + pubsub = createPubSubClient(), + skipPubSub = false ) => { await authTrx( async (tx) => { @@ -128,6 +134,10 @@ export const addLabelsToLibraryItem = async ( userId ) + if (skipPubSub) { + return + } + // create pubsub event await pubsub.entityCreated( EntityType.LABEL, diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 02d60e1d0..3511020ab 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -9,8 +9,8 @@ import { createPubSubClient, EntityType } from '../pubsub' import { authTrx, getColumns } from '../repository' import { libraryItemRepository } from '../repository/library_item' import { SaveFollowingItemRequest } from '../routers/svc/following' -import { SetClaimsRole } from '../utils/dictionary' import { generateSlug, wordsCount } from '../utils/helpers' +import { createThumbnailUrl } from '../utils/imageproxy' import { DateFilter, FieldFilter, @@ -105,9 +105,24 @@ const buildWhereClause = ( } if (args.inFilter !== InFilter.ALL) { - queryBuilder.andWhere('library_item.folder = :folder', { - folder: args.inFilter, - }) + switch (args.inFilter) { + case InFilter.INBOX: + queryBuilder.andWhere('library_item.archived_at IS NULL') + break + case InFilter.ARCHIVE: + queryBuilder.andWhere('library_item.archived_at IS NOT NULL') + break + case InFilter.TRASH: + // return only deleted pages within 14 days + queryBuilder.andWhere( + "library_item.deleted_at >= now() - interval '14 days'" + ) + break + default: + queryBuilder.andWhere('library_item.folder = :folder', { + folder: args.inFilter, + }) + } } if (args.readFilter !== ReadFilter.ALL) { @@ -227,13 +242,13 @@ const buildWhereClause = ( } if (!args.includeDeleted && args.inFilter !== InFilter.TRASH) { - queryBuilder.andWhere("library_item.folder <> 'trash'") + queryBuilder.andWhere("library_item.state <> 'DELETED'") } if (args.noFilters) { args.noFilters.forEach((filter) => { queryBuilder.andWhere( - `library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL` + `(library_item.${filter.field} = '{}' OR library_item.${filter.field} IS NULL)` ) }) } @@ -365,6 +380,8 @@ export const restoreLibraryItem = async ( { state: LibraryItemState.Succeeded, savedAt: new Date(), + archivedAt: null, + deletedAt: null, }, userId, pubsub @@ -380,6 +397,21 @@ export const updateLibraryItem = async ( const updatedLibraryItem = await authTrx( async (tx) => { const itemRepo = tx.withRepository(libraryItemRepository) + + // reset deletedAt and archivedAt + switch (libraryItem.state) { + case LibraryItemState.Archived: + libraryItem.archivedAt = new Date() + break + case LibraryItemState.Deleted: + libraryItem.deletedAt = new Date() + break + case LibraryItemState.Processing: + case LibraryItemState.Succeeded: + libraryItem.archivedAt = null + libraryItem.deletedAt = null + break + } await itemRepo.update(id, libraryItem) return itemRepo.findOneByOrFail({ id }) @@ -519,31 +551,35 @@ export const createLibraryItem = async ( return newLibraryItem } -export const saveFeedItemInFollowing = (input: SaveFollowingItemRequest) => { +export const saveFeedItemInFollowing = ( + input: SaveFollowingItemRequest, + userId: string +) => { + const thumbnail = input.thumbnail && createThumbnailUrl(input.thumbnail) + return authTrx( async (tx) => { - const libraryItems: QueryDeepPartialEntity[] = - input.userIds.map((userId) => ({ - ...input, - user: { id: userId }, - originalUrl: input.url, - subscription: input.addedToFollowingBy, - folder: InFilter.FOLLOWING, - slug: generateSlug(input.title), - })) + const itemToSave: QueryDeepPartialEntity = { + ...input, + user: { id: userId }, + originalUrl: input.url, + subscription: input.addedToFollowingBy, + folder: InFilter.FOLLOWING, + slug: generateSlug(input.title), + thumbnail, + } return tx .getRepository(LibraryItem) .createQueryBuilder() .insert() - .values(libraryItems) + .values(itemToSave) .orIgnore() // ignore if the item already exists .returning('*') .execute() }, undefined, - undefined, - SetClaimsRole.ADMIN + userId ) } @@ -600,14 +636,14 @@ export const updateLibraryItems = async ( switch (action) { case BulkActionType.Archive: values = { - folder: InFilter.ARCHIVE, - savedAt: new Date(), + archivedAt: new Date(), + state: LibraryItemState.Archived, } break case BulkActionType.Delete: values = { - savedAt: new Date(), - folder: InFilter.TRASH, + state: LibraryItemState.Deleted, + deletedAt: new Date(), } break case BulkActionType.AddLabels: diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 5ee1b2708..6eb5ebff3 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -261,6 +261,9 @@ export const parsedContentToLibraryItem = ({ uploadFileId: uploadFileId || undefined, readingProgressTopPercent: 0, readingProgressHighestReadAnchor: 0, + state: state + ? (state as unknown as LibraryItemState) + : LibraryItemState.Succeeded, createdAt: validatedDate(saveTime), savedAt: validatedDate(saveTime), siteName: parsedContent?.siteName, @@ -269,7 +272,8 @@ export const parsedContentToLibraryItem = ({ wordCount: wordsCount(parsedContent?.textContent || ''), contentReader: contentReaderForLibraryItem(itemType, uploadFileId), subscription: rssFeedUrl, - folder: state === ArticleSavingRequestStatus.Archived ? 'archive' : 'inbox', - state: LibraryItemState.Succeeded, + folder: 'inbox', + archivedAt: + state === ArticleSavingRequestStatus.Archived ? new Date() : null, } } diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 80a883be7..fedfe34f0 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -240,7 +240,7 @@ export const libraryItemToArticle = (item: LibraryItem): Article => ({ state: item.state as unknown as ArticleSavingRequestStatus, content: item.readableContent, hash: item.textContentHash || '', - isArchived: item.folder === InFilter.ARCHIVE, + isArchived: !!item.archivedAt, recommendations: item.recommendations?.map( recommandationDataToRecommendation ), @@ -259,7 +259,7 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ url: item.originalUrl, state: item.state as unknown as ArticleSavingRequestStatus, content: item.readableContent, - isArchived: item.folder === InFilter.ARCHIVE, + isArchived: !!item.archivedAt, pageType: item.itemType as unknown as PageType, readingProgressPercent: item.readingProgressBottomPercent, contentReader: item.contentReader as unknown as ContentReader, diff --git a/packages/api/src/utils/imageproxy.ts b/packages/api/src/utils/imageproxy.ts index 104c9b811..f449ca871 100644 --- a/packages/api/src/utils/imageproxy.ts +++ b/packages/api/src/utils/imageproxy.ts @@ -27,3 +27,6 @@ export function createImageProxyUrl( return `${env.imageProxy.url}/${width}x${height},s${signature}/${url}` } + +export const createThumbnailUrl = (url: string): string => + createImageProxyUrl(url, 320, 320) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 6fc8f5f88..e5fb2d5ee 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -410,7 +410,8 @@ describe('Article API', () => { title, user: { id: user.id }, originalUrl: url, - folder: 'archive', + archivedAt: new Date(), + state: LibraryItemState.Archived, }, user.id ) @@ -608,7 +609,7 @@ describe('Article API', () => { ).expect(200) const savedItem = await findLibraryItemByUrl(url, user.id) - expect(savedItem?.folder).to.eql('archive') + expect(savedItem?.archivedAt).to.not.be.null expect(savedItem?.labels?.map((l) => l.name)).to.eql(labels) }) }) @@ -691,7 +692,7 @@ describe('Article API', () => { 200 ) const item = await findLibraryItemById(itemId, user.id) - expect(item?.folder).to.eql('trash') + expect(item?.state).to.eql(LibraryItemState.Deleted) }) }) @@ -1030,7 +1031,8 @@ describe('Article API', () => { readableContent: '

test 1

', slug: 'test slug 1', originalUrl: `${url}/test1`, - folder: 'archive', + archivedAt: new Date(), + state: LibraryItemState.Archived, }, { user, @@ -1038,7 +1040,8 @@ describe('Article API', () => { readableContent: '

test 2

', slug: 'test slug 2', originalUrl: `${url}/test2`, - folder: 'archive', + archivedAt: new Date(), + state: LibraryItemState.Archived, }, { user, @@ -1150,12 +1153,12 @@ describe('Article API', () => { }) }) - context("when in:inbox label:test' is in the query", () => { + context('when in:inbox no:subscription label:test is in the query', () => { let items: LibraryItem[] = [] let label: Label before(async () => { - keyword = 'in:inbox label:test' + keyword = 'in:inbox no:subscription label:test' // Create some test items label = await createLabel('test', '', user.id) items = await createLibraryItems( @@ -1167,19 +1170,29 @@ describe('Article API', () => { slug: 'test slug 1', originalUrl: `${url}/test1`, }, + { + user, + title: 'test title 2', + readableContent: '

test 2

', + slug: 'test slug 2', + originalUrl: `${url}/test2`, + subscription: 'test subscription', + }, { user, title: 'test title 3', readableContent: '

test 3

', slug: 'test slug 3', originalUrl: `${url}/test3`, - folder: 'archive', + archivedAt: new Date(), + state: LibraryItemState.Archived, }, ], user.id ) await saveLabelsInLibraryItem([label], items[0].id, user.id) await saveLabelsInLibraryItem([label], items[1].id, user.id) + await saveLabelsInLibraryItem([label], items[2].id, user.id) }) after(async () => { @@ -1263,7 +1276,7 @@ describe('Article API', () => { slug: 'test slug 1', originalUrl: `${url}/test1`, itemType: PageType.File, - folder: 'archive', + archivedAt: new Date(), }, { user, @@ -1271,7 +1284,7 @@ describe('Article API', () => { readableContent: '

test 2

', slug: 'test slug 2', originalUrl: `${url}/test2`, - folder: 'archive', + archivedAt: new Date(), readingProgressBottomPercent: 100, }, { @@ -1313,7 +1326,7 @@ describe('Article API', () => { slug: 'test slug 1', originalUrl: `${url}/test1`, subscription: 'feed', - folder: 'archive', + archivedAt: new Date(), }, { user, @@ -1329,7 +1342,7 @@ describe('Article API', () => { readableContent: '

test 3

', slug: 'test slug 3', originalUrl: `${url}/test3`, - folder: 'archive', + archivedAt: new Date(), }, ], user.id @@ -1362,7 +1375,7 @@ describe('Article API', () => { readableContent: '

test 1

', slug: 'test slug 1', originalUrl: `${url}/test1`, - folder: 'trash', + deletedAt: new Date(), }, { user, @@ -1371,7 +1384,7 @@ describe('Article API', () => { slug: 'test slug 2', originalUrl: `${url}/test2`, readingProgressBottomPercent: 100, - folder: 'trash', + deletedAt: new Date(), }, { user, @@ -1733,7 +1746,7 @@ describe('Article API', () => { for (let i = 0; i < 3; i++) { await updateLibraryItem( items[i].id, - { folder: 'trash', savedAt: new Date() }, + { state: LibraryItemState.Deleted, deletedAt: new Date() }, user.id ) deletedItems.push(items[i]) diff --git a/packages/db/migrations/0148.do.update_folder_in_library_item.sql b/packages/db/migrations/0148.do.update_folder_in_library_item.sql new file mode 100755 index 000000000..33d0ca9db --- /dev/null +++ b/packages/db/migrations/0148.do.update_folder_in_library_item.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: update_folder_in_library_item +-- Description: Update folder column in library_item table + +BEGIN; + +UPDATE omnivore.library_item SET folder = 'inbox' WHERE folder = 'archive' OR folder = 'trash'; + +COMMIT; diff --git a/packages/db/migrations/0148.undo.update_folder_in_library_item.sql b/packages/db/migrations/0148.undo.update_folder_in_library_item.sql new file mode 100755 index 000000000..3b3eea8c2 --- /dev/null +++ b/packages/db/migrations/0148.undo.update_folder_in_library_item.sql @@ -0,0 +1,10 @@ +-- Type: UNDO +-- Name: update_folder_in_library_item +-- Description: Update folder column in library_item table + +BEGIN; + +UPDATE omnivore.library_item SET folder = 'archive' WHERE archived_at IS NOT NULL; +UPDATE omnivore.library_item SET folder = 'trash' WHERE deleted_at IS NOT NULL; + +COMMIT; diff --git a/packages/integration-handler/src/index.ts b/packages/integration-handler/src/index.ts index bb27136cd..30de57935 100644 --- a/packages/integration-handler/src/index.ts +++ b/packages/integration-handler/src/index.ts @@ -251,33 +251,33 @@ export const importer = Sentry.GCPFunction.wrapHttpFunction( total: offset, size: retrievedData.length, }) - - console.log('updating integration...', { - userId, - integrationId: req.body.integrationId, - syncedAt, - }) - // update the integration's syncedAt and remove taskName - const result = await updateIntegration( - REST_BACKEND_ENDPOINT, - req.body.integrationId, - new Date(syncedAt), - req.body.integrationName, - claims.token, - token, - 'IMPORT', - null - ) - if (!result) { - console.error('failed to update integration', { - userId, - integrationId: req.body.integrationId, - }) - return res.status(400).send('Failed to update integration') - } } while (retrievedData.length > 0 && offset < 20000) // limit to 20k pages } + console.log('updating integration...', { + userId, + integrationId: req.body.integrationId, + syncedAt, + }) + // update the integration's syncedAt and remove taskName + const result = await updateIntegration( + REST_BACKEND_ENDPOINT, + req.body.integrationId, + new Date(syncedAt), + req.body.integrationName, + claims.token, + token, + 'IMPORT', + null + ) + if (!result) { + console.error('failed to update integration', { + userId, + integrationId: req.body.integrationId, + }) + return res.status(500).send('Failed to update integration') + } + console.log('done') } catch (err) { console.error('import pages from integration failed', { diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2c4461c99..75a621ee5 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -19,6 +19,35 @@ interface RssFeedRequest { // link can be a string or an object type RssFeedItemLink = string | { $: { rel?: string; href: string } } +type RssFeed = Parser.Output<{ + published?: string + updated?: string + created?: string + link?: RssFeedItemLink + links?: RssFeedItemLink[] +}> & { + lastBuildDate?: string + 'syn:updatePeriod'?: string + 'syn:updateFrequency'?: string + 'sy:updatePeriod'?: string + 'sy:updateFrequency'?: string +} +type RssFeedItemMedia = { + $: { url: string; width?: string; height?: string; medium?: string } +} +type RssFeedItem = Item & { + 'media:thumbnail'?: RssFeedItemMedia + 'media:content'?: RssFeedItemMedia[] +} + +const getThumbnail = (item: RssFeedItem) => { + if (item['media:thumbnail']) { + return item['media:thumbnail'].$.url + } + + return item['media:content']?.find((media) => media.$.medium === 'image')?.$ + .url +} function isRssFeedRequest(body: any): body is RssFeedRequest { return ( @@ -125,7 +154,7 @@ const sendUpdateSubscriptionMutation = async ( const createTask = async ( userId: string, feedUrl: string, - item: Item, + item: RssFeedItem, autoAddToLibrary: boolean ) => { if (autoAddToLibrary) { @@ -138,7 +167,7 @@ const createTask = async ( const createSavingItemTask = async ( userId: string, feedUrl: string, - item: Item + item: RssFeedItem ) => { const input = { userId, @@ -167,7 +196,7 @@ const createSavingItemTask = async ( const createFollowingTask = async ( userId: string, feedUrl: string, - item: Item + item: RssFeedItem ) => { const input = { userIds: [userId], @@ -181,6 +210,7 @@ const createFollowingTask = async ( savedAt: item.isoDate, publishedAt: item.isoDate, previewContentType: 'text/html', // TODO: get content type from feed + thumbnail: getThumbnail(item), } try { @@ -217,6 +247,8 @@ const parser = new Parser({ 'published', 'updated', 'created', + ['media:content', 'media:content', { keepArray: true }], + ['media:thumbnail'], ], feed: [ 'lastBuildDate', @@ -228,9 +260,9 @@ const parser = new Parser({ }, }) -const getUpdateFrequency = (feed: any) => { - const updateFrequency = (feed['syn:updateFrequency'] || - feed['sy:updateFrequency']) as string | undefined +const getUpdateFrequency = (feed: RssFeed) => { + const updateFrequency = + feed['syn:updateFrequency'] || feed['sy:updateFrequency'] if (!updateFrequency) { return 1 @@ -244,10 +276,8 @@ const getUpdateFrequency = (feed: any) => { return frequency } -const getUpdatePeriodInHours = (feed: any) => { - const updatePeriod = (feed['syn:updatePeriod'] || feed['sy:updatePeriod']) as - | string - | undefined +const getUpdatePeriodInHours = (feed: RssFeed) => { + const updatePeriod = feed['syn:updatePeriod'] || feed['sy:updatePeriod'] switch (updatePeriod) { case 'hourly': @@ -301,19 +331,7 @@ const processSubscription = async ( scheduledAt: number, lastFetchedChecksum: string, autoAddToLibrary: boolean, - feed: { - lastBuildDate: any - 'syn:updatePeriod': any - 'syn:updateFrequency': any - 'sy:updatePeriod': any - 'sy:updateFrequency': any - } & Parser.Output<{ - published: any - updated: any - created: any - link: any - links: any[] - }> + feed: RssFeed ) => { let lastItemFetchedAt: Date | null = null let lastValidItem: Item | null = null @@ -327,7 +345,7 @@ const processSubscription = async ( // fetch feed let itemCount = 0 - const feedLastBuildDate = feed.lastBuildDate as string | undefined + const feedLastBuildDate = feed.lastBuildDate console.log('Feed last build date', feedLastBuildDate) if ( feedLastBuildDate && @@ -341,10 +359,7 @@ const processSubscription = async ( for (const item of feed.items) { // use published or updated if isoDate is not available for atom feeds item.isoDate = - item.isoDate || - (item.published as string) || - (item.updated as string) || - (item.created as string) + item.isoDate || item.published || item.updated || item.created console.log('Processing feed item', item.links, item.isoDate) if (!item.links || item.links.length === 0) { @@ -352,7 +367,7 @@ const processSubscription = async ( continue } - item.link = getLink(item.links as RssFeedItemLink[]) + item.link = getLink(item.links) if (!item.link) { console.log('Invalid feed item links', item.links) continue diff --git a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx index 2149ed68b..0c9000ea2 100644 --- a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx +++ b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx @@ -242,9 +242,9 @@ function Subscriptions( {!collapsed ? ( <> - +