diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index fcb1e2d96..601bbfb0c 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -34,7 +34,10 @@ import { authorized } from '../../utils/helpers' type PartialSubscription = Omit -const parser = new Parser() +const parser = new Parser({ + timeout: 30000, // 30 seconds + maxRedirects: 5, +}) export type SubscriptionsSuccessPartial = Merge< SubscriptionsSuccess, diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 49abc27a3..a44686dc3 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -12,6 +12,14 @@ interface RssFeedRequest { lastFetchedAt: number // unix timestamp in milliseconds } +interface RssFeedItemLink { + $: { + rel?: string + href: string + type?: string + } +} + function isRssFeedRequest(body: any): body is RssFeedRequest { return ( 'subscriptionId' in body && 'feedUrl' in body && 'lastFetchedAt' in body @@ -114,7 +122,32 @@ Sentry.GCPFunction.init({ }) const signToken = promisify(jwt.sign) -const parser = new Parser() +const parser = new Parser({ + customFields: { + item: [['link', 'links', { keepArray: true }]], + }, +}) + +// get link following the order of preference: via, self, alternate +const getLink = (links: RssFeedItemLink[]) => { + // sort links by preference + const sortedLinks: string[] = [] + + links.forEach((link) => { + if (link.$.rel === 'via') { + sortedLinks[0] = link.$.href + } + if (link.$.rel === 'self' || !link.$.rel) { + sortedLinks[1] = link.$.href + } + if (link.$.rel === 'alternate') { + sortedLinks[2] = link.$.href + } + }) + + // return the first link that is not undefined + return sortedLinks.find((link) => !!link) +} export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { @@ -159,13 +192,21 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( // save each item in the feed for (const item of feed.items) { - console.log('Processing feed item', item.link, item.isoDate) + console.log('Processing feed item', item.links, item.isoDate) - if (!item.link) { + if (!item.links || item.links.length === 0) { console.log('Invalid feed item', item) continue } + item.link = getLink(item.links as RssFeedItemLink[]) + if (!item.link) { + console.log('Invalid feed item links', item.links) + continue + } + + console.log('Fetching feed item', item.link) + const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() // remember the last valid item if (