From c9b93a118011784ccd5ac433f4dfedc3793f0774 Mon Sep 17 00:00:00 2001
From: Jackson Harper <jacksonh@gmail.com>
Date: Fri, 29 Dec 2023 12:26:43 +0800
Subject: [PATCH] Dont try to fetch content for some feed URLs like arvix which
 blocks fetches

---
 packages/rss-handler/src/index.ts | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts
index 067a55d54..fcbd3b9d8 100644
--- a/packages/rss-handler/src/index.ts
+++ b/packages/rss-handler/src/index.ts
@@ -53,6 +53,13 @@ export const isOldItem = (item: RssFeedItem, lastFetchedAt: number) => {
   )
 }
 
+export const isContentFetchBlocked = (feedUrl: string) => {
+  if (feedUrl.startsWith('https://arxiv.org/')) {
+    return true
+  }
+  return false
+}
+
 const getThumbnail = (item: RssFeedItem) => {
   if (item['media:thumbnail']) {
     return item['media:thumbnail'].$.url
@@ -455,6 +462,10 @@ const processSubscription = async (
       continue
     }
 
+    if (isContentFetchBlocked(feedUrl)) {
+      fetchContent = false
+    }
+
     const created = await createTask(
       userId,
       feedUrl,