diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 61fab2be5..57bd61997 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2301,7 +2301,6 @@ export enum ScanFeedsErrorCode { export type ScanFeedsInput = { opml?: InputMaybe; - type: ScanFeedsType; url?: InputMaybe; }; @@ -2312,11 +2311,6 @@ export type ScanFeedsSuccess = { feeds: Array; }; -export enum ScanFeedsType { - Html = 'HTML', - Opml = 'OPML' -} - export type SearchError = { __typename?: 'SearchError'; errorCodes: Array; @@ -3790,7 +3784,6 @@ export type ResolversTypes = { ScanFeedsInput: ScanFeedsInput; ScanFeedsResult: ResolversTypes['ScanFeedsError'] | ResolversTypes['ScanFeedsSuccess']; ScanFeedsSuccess: ResolverTypeWrapper; - ScanFeedsType: ScanFeedsType; SearchError: ResolverTypeWrapper; SearchErrorCode: SearchErrorCode; SearchItem: ResolverTypeWrapper; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 62e177259..8f64a27b7 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1751,7 +1751,6 @@ enum ScanFeedsErrorCode { input ScanFeedsInput { opml: String - type: ScanFeedsType! url: String } @@ -1761,11 +1760,6 @@ type ScanFeedsSuccess { feeds: [Feed!]! } -enum ScanFeedsType { - HTML - OPML -} - type SearchError { errorCodes: [SearchErrorCode!]! } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 77464dbec..fad1714d5 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -17,7 +17,6 @@ import { ScanFeedsError, ScanFeedsErrorCode, ScanFeedsSuccess, - ScanFeedsType, SortBy, SortOrder, SubscribeError, @@ -42,7 +41,7 @@ import { Merge } from '../../util' import { analytics } from '../../utils/analytics' import { enqueueRssFeedFetch } from '../../utils/createTask' import { authorized } from '../../utils/helpers' -import { parseFeed, parseOpml } from '../../utils/parser' +import { parseFeed, parseOpml, RSS_PARSER_CONFIG } from '../../utils/parser' type PartialSubscription = Omit @@ -399,22 +398,17 @@ export const scanFeedsResolver = authorized< ScanFeedsSuccess, ScanFeedsError, QueryScanFeedsArgs ->(async (_, { input: { type, opml, url } }, { log, uid }) => { +>(async (_, { input: { opml, url } }, { log, uid }) => { analytics.track({ userId: uid, event: 'scan_feeds', properties: { - type, + opml, + url, }, }) - if (type === ScanFeedsType.Opml) { - if (!opml) { - return { - errorCodes: [ScanFeedsErrorCode.BadRequest], - } - } - + if (opml) { // parse opml const feeds = parseOpml(opml) if (!feeds) { @@ -434,37 +428,52 @@ export const scanFeedsResolver = authorized< } if (!url) { + log.error('Missing opml and url') + return { errorCodes: [ScanFeedsErrorCode.BadRequest], } } try { - // fetch HTML and parse feeds - const response = await axios.get(url, { - timeout: 5000, - headers: { - 'User-Agent': 'Mozilla/5.0', - Accept: 'text/html', - }, - }) - const html = response.data as string - const dom = parseHTML(html).document - const links = dom.querySelectorAll('link[type="application/rss+xml"]') - const feeds = Array.from(links) - .map((link) => ({ - url: link.getAttribute('href') || '', - title: link.getAttribute('title') || '', - type: 'rss', - })) - .filter((feed) => feed.url) + // fetch page content and parse feeds + const response = await axios.get(url, RSS_PARSER_CONFIG) + const content = response.data as string + // check if the content is html or xml + const contentType = response.headers['content-type'] + const isHtml = contentType?.includes('text/html') + if (isHtml) { + // this is an html page, parse rss feed links + const dom = parseHTML(content).document + const links = dom.querySelectorAll('link[type="application/rss+xml"]') + const feeds = Array.from(links) + .map((link) => ({ + url: link.getAttribute('href') || '', + title: link.getAttribute('title') || '', + type: 'rss', + })) + .filter((feed) => feed.url) + + return { + __typename: 'ScanFeedsSuccess', + feeds, + } + } + + // this is the url to an RSS feed + const feed = await parseFeed(url) + if (!feed) { + return { + errorCodes: [ScanFeedsErrorCode.BadRequest], + } + } return { __typename: 'ScanFeedsSuccess', - feeds, + feeds: [feed], } } catch (error) { - log.error('Error scanning HTML', error) + log.error('Error scanning URL', error) return { errorCodes: [ScanFeedsErrorCode.BadRequest], diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 0b4e16ae6..065e1739b 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2686,16 +2686,10 @@ const schema = gql` } input ScanFeedsInput { - type: ScanFeedsType! url: String opml: String } - enum ScanFeedsType { - OPML - HTML - } - union ScanFeedsResult = ScanFeedsSuccess | ScanFeedsError type ScanFeedsSuccess { diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index eb69c8a87..2a8dbb86d 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -75,6 +75,16 @@ const DOM_PURIFY_CONFIG = { const ARTICLE_PREFIX = 'omnivore:' export const FAKE_URL_PREFIX = 'https://omnivore.app/no_url?q=' +export const RSS_PARSER_CONFIG = { + timeout: 5000, // 5 seconds + headers: { + // some rss feeds require user agent + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + Accept: + 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4, text/html;q=0.2', + }, +} /** Hook that prevents DOMPurify from removing youtube iframes */ const domPurifySanitizeHook = ( @@ -812,16 +822,7 @@ export const parseFeed = async (url: string): Promise => { } } - const parser = new Parser({ - timeout: 5000, // 5 seconds - headers: { - // some rss feeds require user agent - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - Accept: - 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', - }, - }) + const parser = new Parser(RSS_PARSER_CONFIG) const feed = await parser.parseURL(url) const feedUrl = feed.feedUrl || url