diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 7ac91c69c..5ca6cd557 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -21,8 +21,12 @@ import { CreateArticleError, CreateArticleErrorCode, CreateArticleSuccess, + MoveToFolderError, + MoveToFolderErrorCode, + MoveToFolderSuccess, MutationBulkActionArgs, MutationCreateArticleArgs, + MutationMoveToFolderArgs, MutationSaveArticleReadingProgressArgs, MutationSetBookmarkArticleArgs, MutationSetFavoriteArticleArgs, @@ -85,6 +89,7 @@ import { generateSlug, isParsingTimeout, libraryItemToArticle, + libraryItemToArticleSavingRequest, libraryItemToSearchItem, titleForFilePath, userDataToUser, @@ -870,6 +875,80 @@ export const setFavoriteArticleResolver = authorized< } }) +export const moveToFolderResolver = authorized< + MoveToFolderSuccess, + MoveToFolderError, + MutationMoveToFolderArgs +>(async (_, { id, folder }, { authTrx, pubsub, uid }) => { + analytics.track({ + userId: uid, + event: 'move_to_folder', + properties: { + id, + folder, + }, + }) + + const item = await authTrx((tx) => + tx.getRepository(LibraryItem).findOne({ + where: { + id, + }, + relations: ['user'], + }) + ) + + if (!item) { + return { + errorCodes: [MoveToFolderErrorCode.Unauthorized], + } + } + + if (item.folder === folder) { + return { + errorCodes: [MoveToFolderErrorCode.AlreadyExists], + } + } + + const savedAt = new Date() + + // if the content is not fetched yet, create a page save request + if (!item.readableContent) { + const articleSavingRequest = await createPageSaveRequest({ + userId: uid, + url: item.originalUrl, + articleSavingRequestId: id, + priority: 'high', + publishedAt: item.publishedAt || undefined, + savedAt, + pubsub, + }) + + return { + __typename: 'MoveToFolderSuccess', + articleSavingRequest, + } + } + + const updatedItem = await updateLibraryItem( + item.id, + { + folder, + savedAt, + }, + uid, + pubsub + ) + + return { + __typename: 'MoveToFolderSuccess', + articleSavingRequest: libraryItemToArticleSavingRequest( + updatedItem.user, + updatedItem + ), + } +}) + const getUpdateReason = (libraryItem: LibraryItem, since: Date) => { if (libraryItem.deletedAt) { return UpdateReason.Deleted diff --git a/packages/api/src/resolvers/following/index.ts b/packages/api/src/resolvers/following/index.ts deleted file mode 100644 index 2d20e3d24..000000000 --- a/packages/api/src/resolvers/following/index.ts +++ /dev/null @@ -1,231 +0,0 @@ -import axios from 'axios' -import { parseHTML } from 'linkedom' -import { LibraryItem } from '../../entity/library_item' -import { - FeedEdge, - FeedsError, - FeedsErrorCode, - FeedsSuccess, - MoveToFolderError, - MoveToFolderErrorCode, - MoveToFolderSuccess, - MutationMoveToFolderArgs, - QueryFeedsArgs, - QueryScanFeedsArgs, - ScanFeedsError, - ScanFeedsErrorCode, - ScanFeedsSuccess, - ScanFeedsType, -} from '../../generated/graphql' -import { feedRepository } from '../../repository/feed' -import { createPageSaveRequest } from '../../services/create_page_save_request' -import { updateLibraryItem } from '../../services/library_item' -import { analytics } from '../../utils/analytics' -import { - authorized, - libraryItemToArticleSavingRequest, -} from '../../utils/helpers' -import { parseOpml } from '../../utils/parser' - -export const feedsResolver = authorized< - FeedsSuccess, - FeedsError, - QueryFeedsArgs ->(async (_, { input }, { log }) => { - try { - const startCursor = input.after || '' - const start = - startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 - const first = Math.min(input.first || 10, 100) // cap at 100 - - const { feeds, count } = await feedRepository.searchFeeds( - input.query || '', - first + 1, // fetch one extra to check if there is a next page - start, - input.sort?.by, - input.sort?.order || undefined - ) - - const hasNextPage = feeds.length > first - const endCursor = String(start + feeds.length - (hasNextPage ? 1 : 0)) - - if (hasNextPage) { - // remove an extra if exists - feeds.pop() - } - - const edges: FeedEdge[] = feeds.map((feed) => ({ - node: feed, - cursor: endCursor, - })) - - return { - __typename: 'FeedsSuccess', - edges, - pageInfo: { - hasPreviousPage: start > 0, - hasNextPage, - startCursor, - endCursor, - totalCount: count, - }, - } - } catch (error) { - log.error('Error fetching feeds', error) - - return { - errorCodes: [FeedsErrorCode.BadRequest], - } - } -}) - -export const moveToFolderResolver = authorized< - MoveToFolderSuccess, - MoveToFolderError, - MutationMoveToFolderArgs ->(async (_, { id, folder }, { authTrx, pubsub, uid }) => { - analytics.track({ - userId: uid, - event: 'move_to_folder', - properties: { - id, - folder, - }, - }) - - const item = await authTrx((tx) => - tx.getRepository(LibraryItem).findOne({ - where: { - id, - }, - relations: ['user'], - }) - ) - - if (!item) { - return { - errorCodes: [MoveToFolderErrorCode.Unauthorized], - } - } - - if (item.folder === folder) { - return { - errorCodes: [MoveToFolderErrorCode.AlreadyExists], - } - } - - const savedAt = new Date() - - // if the content is not fetched yet, create a page save request - if (!item.readableContent) { - const articleSavingRequest = await createPageSaveRequest({ - userId: uid, - url: item.originalUrl, - articleSavingRequestId: id, - priority: 'high', - publishedAt: item.publishedAt || undefined, - savedAt, - pubsub, - }) - - return { - __typename: 'MoveToFolderSuccess', - articleSavingRequest, - } - } - - const updatedItem = await updateLibraryItem( - item.id, - { - folder, - savedAt, - }, - uid, - pubsub - ) - - return { - __typename: 'MoveToFolderSuccess', - articleSavingRequest: libraryItemToArticleSavingRequest( - updatedItem.user, - updatedItem - ), - } -}) - -export const scanFeedsResolver = authorized< - ScanFeedsSuccess, - ScanFeedsError, - QueryScanFeedsArgs ->(async (_, { input: { type, opml, url } }, { log, uid }) => { - analytics.track({ - userId: uid, - event: 'scan_feeds', - properties: { - type, - }, - }) - - if (type === ScanFeedsType.Opml) { - if (!opml) { - return { - errorCodes: [ScanFeedsErrorCode.BadRequest], - } - } - - // parse opml - const feeds = parseOpml(opml) - if (!feeds) { - return { - errorCodes: [ScanFeedsErrorCode.BadRequest], - } - } - - return { - __typename: 'ScanFeedsSuccess', - feeds: feeds.map((feed) => ({ - url: feed.feedUrl, - title: feed.title, - type: feed.feedType || 'rss', - })), - } - } - - if (!url) { - return { - errorCodes: [ScanFeedsErrorCode.BadRequest], - } - } - - try { - // fetch HTML and parse feeds - const response = await axios.get(url, { - timeout: 5000, - headers: { - 'User-Agent': 'Mozilla/5.0', - Accept: 'text/html', - }, - }) - const html = response.data as string - const dom = parseHTML(html).document - const links = dom.querySelectorAll('link[type="application/rss+xml"]') - const feeds = Array.from(links) - .map((link) => ({ - url: link.getAttribute('href') || '', - title: link.getAttribute('title') || '', - type: 'rss', - })) - .filter((feed) => feed.url) - - return { - __typename: 'ScanFeedsSuccess', - feeds, - } - } catch (error) { - log.error('Error scanning HTML', error) - - return { - errorCodes: [ScanFeedsErrorCode.BadRequest], - } - } -}) diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index ff363e78f..909aeb121 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -29,7 +29,6 @@ import { generateUploadFilePathName, } from '../utils/uploads' import { optInFeatureResolver } from './features' -import { feedsResolver, moveToFolderResolver } from './following' import { uploadImportFileResolver } from './importers/uploadImportFileResolver' import { addPopularReadResolver, @@ -53,6 +52,7 @@ import { deleteRuleResolver, deleteWebhookResolver, deviceTokensResolver, + feedsResolver, filtersResolver, generateApiKeyResolver, getAllUsersResolver, @@ -76,6 +76,7 @@ import { mergeHighlightResolver, moveFilterResolver, moveLabelResolver, + moveToFolderResolver, newsletterEmailsResolver, recommendHighlightsResolver, recommendResolver, @@ -88,6 +89,7 @@ import { saveFilterResolver, savePageResolver, saveUrlResolver, + scanFeedsResolver, searchResolver, sendInstallInstructionsResolver, setBookmarkArticleResolver, @@ -249,6 +251,7 @@ export const functionResolvers = { groups: groupsResolver, recentEmails: recentEmailsResolver, feeds: feedsResolver, + scanFeeds: scanFeedsResolver, }, User: { async intercomHash( diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 0680339ff..98e0d9c79 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -1,12 +1,24 @@ +import axios from 'axios' +import { parseHTML } from 'linkedom' import Parser from 'rss-parser' import { Brackets } from 'typeorm' import { Subscription } from '../../entity/subscription' import { env } from '../../env' import { + FeedEdge, + FeedsError, + FeedsErrorCode, + FeedsSuccess, MutationSubscribeArgs, MutationUnsubscribeArgs, MutationUpdateSubscriptionArgs, + QueryFeedsArgs, + QueryScanFeedsArgs, QuerySubscriptionsArgs, + ScanFeedsError, + ScanFeedsErrorCode, + ScanFeedsSuccess, + ScanFeedsType, SortBy, SortOrder, SubscribeError, @@ -25,11 +37,13 @@ import { UpdateSubscriptionSuccess, } from '../../generated/graphql' import { getRepository } from '../../repository' +import { feedRepository } from '../../repository/feed' import { unsubscribe } from '../../services/subscriptions' import { Merge } from '../../util' import { analytics } from '../../utils/analytics' import { enqueueRssFeedFetch } from '../../utils/createTask' import { authorized } from '../../utils/helpers' +import { parseFeed, parseOpml } from '../../utils/parser' type PartialSubscription = Omit @@ -175,7 +189,7 @@ export const subscribeResolver = authorized< SubscribeSuccessPartial, SubscribeError, MutationSubscribeArgs ->(async (_, { input }, { authTrx, uid, log }) => { +>(async (_, { input }, { uid, log }) => { try { analytics.track({ userId: uid, @@ -224,7 +238,12 @@ export const subscribeResolver = authorized< // create new rss subscription const MAX_RSS_SUBSCRIPTIONS = 150 // validate rss feed - const feed = await parser.parseURL(input.url) + const feed = await parseFeed(input.url) + if (!feed) { + return { + errorCodes: [SubscribeErrorCode.NotFound], + } + } // limit number of rss subscriptions to 150 const results = (await getRepository(Subscription).query( @@ -235,11 +254,11 @@ export const subscribeResolver = authorized< returning *;`, [ feed.title, - feed.feedUrl, + feed.url, feed.description || null, SubscriptionType.Rss, uid, - feed.image?.url || null, + feed.thumbnail || null, input.autoAddToLibrary ?? null, input.isPrivate ?? null, MAX_RSS_SUBSCRIPTIONS, @@ -336,3 +355,132 @@ export const updateSubscriptionResolver = authorized< } } }) + +export const feedsResolver = authorized< + FeedsSuccess, + FeedsError, + QueryFeedsArgs +>(async (_, { input }, { log }) => { + try { + const startCursor = input.after || '' + const start = + startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0 + const first = Math.min(input.first || 10, 100) // cap at 100 + + const { feeds, count } = await feedRepository.searchFeeds( + input.query || '', + first + 1, // fetch one extra to check if there is a next page + start, + input.sort?.by, + input.sort?.order || undefined + ) + + const hasNextPage = feeds.length > first + const endCursor = String(start + feeds.length - (hasNextPage ? 1 : 0)) + + if (hasNextPage) { + // remove an extra if exists + feeds.pop() + } + + const edges: FeedEdge[] = feeds.map((feed) => ({ + node: feed, + cursor: endCursor, + })) + + return { + __typename: 'FeedsSuccess', + edges, + pageInfo: { + hasPreviousPage: start > 0, + hasNextPage, + startCursor, + endCursor, + totalCount: count, + }, + } + } catch (error) { + log.error('Error fetching feeds', error) + + return { + errorCodes: [FeedsErrorCode.BadRequest], + } + } +}) + +export const scanFeedsResolver = authorized< + ScanFeedsSuccess, + ScanFeedsError, + QueryScanFeedsArgs +>(async (_, { input: { type, opml, url } }, { log, uid }) => { + analytics.track({ + userId: uid, + event: 'scan_feeds', + properties: { + type, + }, + }) + + if (type === ScanFeedsType.Opml) { + if (!opml) { + return { + errorCodes: [ScanFeedsErrorCode.BadRequest], + } + } + + // parse opml + const feeds = parseOpml(opml) + if (!feeds) { + return { + errorCodes: [ScanFeedsErrorCode.BadRequest], + } + } + + return { + __typename: 'ScanFeedsSuccess', + feeds: feeds.map((feed) => ({ + url: feed.url, + title: feed.title, + type: feed.type || 'rss', + })), + } + } + + if (!url) { + return { + errorCodes: [ScanFeedsErrorCode.BadRequest], + } + } + + try { + // fetch HTML and parse feeds + const response = await axios.get(url, { + timeout: 5000, + headers: { + 'User-Agent': 'Mozilla/5.0', + Accept: 'text/html', + }, + }) + const html = response.data as string + const dom = parseHTML(html).document + const links = dom.querySelectorAll('link[type="application/rss+xml"]') + const feeds = Array.from(links) + .map((link) => ({ + url: link.getAttribute('href') || '', + title: link.getAttribute('title') || '', + type: 'rss', + })) + .filter((feed) => feed.url) + + return { + __typename: 'ScanFeedsSuccess', + feeds, + } + } catch (error) { + log.error('Error scanning HTML', error) + + return { + errorCodes: [ScanFeedsErrorCode.BadRequest], + } + } +}) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index d1516a9c9..fc6d722cc 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -12,6 +12,7 @@ import * as jwt from 'jsonwebtoken' import { parseHTML } from 'linkedom' import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown' import { ElementNode } from 'node-html-markdown/dist/nodes' +import Parser from 'rss-parser' import { parser } from 'sax' import { ILike } from 'typeorm' import { promisify } from 'util' @@ -35,13 +36,23 @@ import { buildLogger, LogRecord } from './logger' interface Feed { title: string url: string - feedUrl: string - feedType: string + type: string + thumbnail?: string + description?: string } const logger = buildLogger('utils.parse') const signToken = promisify(jwt.sign) +const axiosInstance = axios.create({ + timeout: 5000, + headers: { + 'User-Agent': 'Mozilla/5.0', + Accept: 'text/html', + }, + responseType: 'text', +}) + export const ALLOWED_CONTENT_TYPES = [ 'text/html', 'application/octet-stream', @@ -712,6 +723,16 @@ export const getDistillerResult = async ( } } +const fetchHtml = async (url: string): Promise => { + try { + const response = await axiosInstance.get(url) + return response.data as string + } catch (error) { + logger.error('Error fetching html', error) + return undefined + } +} + export const parseOpml = (opml: string): Feed[] | undefined => { const xmlParser = parser(true, { lowercase: true }) const feeds: Feed[] = [] @@ -723,13 +744,9 @@ export const parseOpml = (opml: string): Feed[] | undefined => { const feedUrl = node.attributes.xmlUrl.toString() if (feedUrl && !existingFeeds.has(feedUrl)) { feeds.push({ - title: - node.attributes.title.toString() || - node.attributes.text.toString() || - node.attributes.description.toString(), - url: node.attributes.htmlUrl.toString(), - feedUrl: feedUrl.toString(), - feedType: node.attributes.type.toString(), + title: node.attributes.title.toString() || '', + url: feedUrl, + type: node.attributes.type.toString() || 'rss', }) existingFeeds.set(feedUrl, true) } @@ -747,3 +764,77 @@ export const parseOpml = (opml: string): Feed[] | undefined => { return undefined } } + +export const parseHtml = async (url: string): Promise => { + // fetch HTML and parse feeds + const html = await fetchHtml(url) + if (!html) return undefined + + try { + const dom = parseHTML(html).document + const links = dom.querySelectorAll('link[type="application/rss+xml"]') + const feeds = Array.from(links) + .map((link) => ({ + url: link.getAttribute('href') || '', + title: link.getAttribute('title') || '', + type: 'rss', + })) + .filter((feed) => feed.url) + + return feeds + } catch (error) { + logger.error('Error parsing html', error) + return undefined + } +} + +export const parseFeed = async (url: string): Promise => { + try { + // check if url is a telegram channel + const telegramRegex = /https:\/\/t\.me\/([a-zA-Z0-9_]+)/ + const telegramMatch = url.match(telegramRegex) + if (telegramMatch) { + // fetch HTML and parse feeds + const html = await fetchHtml(url) + if (!html) return undefined + + const dom = parseHTML(html).document + const title = dom.querySelector('meta[property="og:title"]') + const thumbnail = dom.querySelector('meta[property="og:image"]') + const description = dom.querySelector('meta[property="og:description"]') + + return { + title: title?.getAttribute('content') || url, + url, + type: 'telegram', + thumbnail: thumbnail?.getAttribute('content') || '', + description: description?.getAttribute('content') || '', + } + } + + const parser = new Parser({ + timeout: 5000, // 5 seconds + headers: { + // some rss feeds require user agent + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + Accept: + 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', + }, + }) + + const feed = await parser.parseURL(url) + const feedUrl = feed.feedUrl || url + + return { + title: feed.title || feedUrl, + url: feedUrl, + thumbnail: feed.image?.url, + type: 'rss', + description: feed.description, + } + } catch (error) { + logger.error('Error parsing feed', error) + return undefined + } +}