From 82d820ea8f341d74cc6125475a8a3a67e8e98ace Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 3 Aug 2023 13:05:26 +0800
Subject: [PATCH 1/3] get link following the order of preference: via, self,
 alternate

---
 .../api/src/resolvers/subscriptions/index.ts  |  4 +-
 packages/rss-handler/src/index.ts             | 43 +++++++++++++++++--
 2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts
index fcb1e2d96..6cff3e8ce 100644
--- a/packages/api/src/resolvers/subscriptions/index.ts
+++ b/packages/api/src/resolvers/subscriptions/index.ts
@@ -34,7 +34,9 @@ import { authorized } from '../../utils/helpers'
 
 type PartialSubscription = Omit<Subscription, 'newsletterEmail'>
 
-const parser = new Parser()
+const parser = new Parser({
+  timeout: 5000, // 5 seconds
+})
 
 export type SubscriptionsSuccessPartial = Merge<
   SubscriptionsSuccess,
diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts
index 49abc27a3..7cebf59db 100644
--- a/packages/rss-handler/src/index.ts
+++ b/packages/rss-handler/src/index.ts
@@ -12,6 +12,12 @@ interface RssFeedRequest {
   lastFetchedAt: number // unix timestamp in milliseconds
 }
 
+interface RssFeedItemLink {
+  rel?: string
+  href: string
+  type?: string
+}
+
 function isRssFeedRequest(body: any): body is RssFeedRequest {
   return (
     'subscriptionId' in body && 'feedUrl' in body && 'lastFetchedAt' in body
@@ -114,7 +120,32 @@ Sentry.GCPFunction.init({
 })
 
 const signToken = promisify(jwt.sign)
-const parser = new Parser()
+const parser = new Parser({
+  customFields: {
+    item: [['link', 'links', { keepArray: true }]],
+  },
+})
+
+// get link following the order of preference: via, self, alternate
+const getLink = (links: RssFeedItemLink[]) => {
+  // sort links by preference
+  const sortedLinks: string[] = []
+
+  links.forEach((link) => {
+    if (link.rel === 'via') {
+      sortedLinks[0] = link.href
+    }
+    if (link.rel === 'self' || !link.rel) {
+      sortedLinks[1] = link.href
+    }
+    if (link.rel === 'alternate') {
+      sortedLinks[2] = link.href
+    }
+  })
+
+  // return the first link that is not undefined
+  return sortedLinks.find((link) => !!link)
+}
 
 export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
   async (req, res) => {
@@ -159,13 +190,19 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       // save each item in the feed
       for (const item of feed.items) {
-        console.log('Processing feed item', item.link, item.isoDate)
+        console.log('Processing feed item', item.links, item.isoDate)
 
-        if (!item.link) {
+        if (!item.links || item.links.length === 0) {
           console.log('Invalid feed item', item)
           continue
         }
 
+        item.link = getLink(item.links as RssFeedItemLink[])
+        if (!item.link) {
+          console.log('Invalid feed item links', item.links)
+          continue
+        }
+
         const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date()
         // remember the last valid item
         if (

From 0da4494772c48f483ad800e188af6983193fa147 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 3 Aug 2023 13:22:03 +0800
Subject: [PATCH 2/3] fix link parsing error

---
 packages/rss-handler/src/index.ts | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts
index 7cebf59db..a44686dc3 100644
--- a/packages/rss-handler/src/index.ts
+++ b/packages/rss-handler/src/index.ts
@@ -13,9 +13,11 @@ interface RssFeedRequest {
 }
 
 interface RssFeedItemLink {
-  rel?: string
-  href: string
-  type?: string
+  $: {
+    rel?: string
+    href: string
+    type?: string
+  }
 }
 
 function isRssFeedRequest(body: any): body is RssFeedRequest {
@@ -132,14 +134,14 @@ const getLink = (links: RssFeedItemLink[]) => {
   const sortedLinks: string[] = []
 
   links.forEach((link) => {
-    if (link.rel === 'via') {
-      sortedLinks[0] = link.href
+    if (link.$.rel === 'via') {
+      sortedLinks[0] = link.$.href
     }
-    if (link.rel === 'self' || !link.rel) {
-      sortedLinks[1] = link.href
+    if (link.$.rel === 'self' || !link.$.rel) {
+      sortedLinks[1] = link.$.href
     }
-    if (link.rel === 'alternate') {
-      sortedLinks[2] = link.href
+    if (link.$.rel === 'alternate') {
+      sortedLinks[2] = link.$.href
     }
   })
 
@@ -203,6 +205,8 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
           continue
         }
 
+        console.log('Fetching feed item', item.link)
+
         const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date()
         // remember the last valid item
         if (

From f4e537a4c75a7f8f7f66bc6cf0b8162cf5a53984 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 3 Aug 2023 13:31:18 +0800
Subject: [PATCH 3/3] set max redirect = 5 and timeout = 30 seconds

---
 packages/api/src/resolvers/subscriptions/index.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts
index 6cff3e8ce..601bbfb0c 100644
--- a/packages/api/src/resolvers/subscriptions/index.ts
+++ b/packages/api/src/resolvers/subscriptions/index.ts
@@ -35,7 +35,8 @@ import { authorized } from '../../utils/helpers'
 type PartialSubscription = Omit<Subscription, 'newsletterEmail'>
 
 const parser = new Parser({
-  timeout: 5000, // 5 seconds
+  timeout: 30000, // 30 seconds
+  maxRedirects: 5,
 })
 
 export type SubscriptionsSuccessPartial = Merge<