prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #2576 from omnivore-app/fix/rss-alternate-link

Browse source 
get link following the order of preference: via, self, alternate
This commit is contained in:
Hongbo Wu 2023-08-03 13:32:19 +08:00 committed by GitHub
commit 859ef4575f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 48 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
packages/api/src/resolvers/subscriptions/index.ts
View file

@ -34,7 +34,10 @@ import { authorized } from '../../utils/helpers'
type PartialSubscription = Omit<Subscription, 'newsletterEmail'>
const parser = new Parser()
const parser = new Parser({
timeout: 30000, // 30 seconds
maxRedirects: 5,
})
export type SubscriptionsSuccessPartial = Merge<
SubscriptionsSuccess,

47
packages/rss-handler/src/index.ts
View file

@ -12,6 +12,14 @@ interface RssFeedRequest {
lastFetchedAt: number // unix timestamp in milliseconds
}
interface RssFeedItemLink {
$: {
rel?: string
href: string
type?: string
}
}
function isRssFeedRequest(body: any): body is RssFeedRequest {
return (
'subscriptionId' in body && 'feedUrl' in body && 'lastFetchedAt' in body
@ -114,7 +122,32 @@ Sentry.GCPFunction.init({
})
const signToken = promisify(jwt.sign)
const parser = new Parser()
const parser = new Parser({
customFields: {
item: [['link', 'links', { keepArray: true }]],
},
})
// get link following the order of preference: via, self, alternate
const getLink = (links: RssFeedItemLink[]) => {
// sort links by preference
const sortedLinks: string[] = []
links.forEach((link) => {
if (link.$.rel === 'via') {
sortedLinks[0] = link.$.href
}
if (link.$.rel === 'self' || !link.$.rel) {
sortedLinks[1] = link.$.href
}
if (link.$.rel === 'alternate') {
sortedLinks[2] = link.$.href
}
})
// return the first link that is not undefined
return sortedLinks.find((link) => !!link)
}
export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
async (req, res) => {
@ -159,13 +192,21 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
// save each item in the feed
for (const item of feed.items) {
console.log('Processing feed item', item.link, item.isoDate)
console.log('Processing feed item', item.links, item.isoDate)
if (!item.link) {
if (!item.links || item.links.length === 0) {
console.log('Invalid feed item', item)
continue
}
item.link = getLink(item.links as RssFeedItemLink[])
if (!item.link) {
console.log('Invalid feed item links', item.links)
continue
}
console.log('Fetching feed item', item.link)
const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date()
// remember the last valid item
if (