saving posts from telegram channel as rss items

packages/api/src/utils/parser.ts

@@ -723,13 +723,13 @@ export const getDistillerResult = async (
   }
 }
 
-const fetchHtml = async (url: string): Promise<string | undefined> => {
+const fetchHtml = async (url: string): Promise<string | null> => {
   try {
     const response = await axiosInstance.get(url)
     return response.data as string
   } catch (error) {
     logger.error('Error fetching html', error)
-    return undefined
+    return null
   }
 }
 
@@ -788,7 +788,7 @@ export const parseHtml = async (url: string): Promise<Feed[] | undefined> => {
   }
 }
 
-export const parseFeed = async (url: string): Promise<Feed | undefined> => {
+export const parseFeed = async (url: string): Promise<Feed | null> => {
   try {
     // check if url is a telegram channel
     const telegramRegex = /https:\/\/t\.me\/([a-zA-Z0-9_]+)/
@@ -796,7 +796,7 @@ export const parseFeed = async (url: string): Promise<Feed | undefined> => {
     if (telegramMatch) {
       // fetch HTML and parse feeds
       const html = await fetchHtml(url)
-      if (!html) return undefined
+      if (!html) return null
 
       const dom = parseHTML(html).document
       const title = dom.querySelector('meta[property="og:title"]')
@@ -835,6 +835,6 @@ export const parseFeed = async (url: string): Promise<Feed | undefined> => {
     }
   } catch (error) {
     logger.error('Error parsing feed', error)
-    return undefined
+    return null
   }
 }

packages/rss-handler/package.json

@@ -28,6 +28,7 @@
     "axios": "^1.4.0",
     "dotenv": "^16.0.1",
     "jsonwebtoken": "^8.5.1",
+    "linkedom": "^0.16.4",
     "rss-parser": "^3.13.0"
   },
   "volta": {

packages/rss-handler/src/index.ts

@@ -3,6 +3,7 @@ import axios from 'axios'
 import crypto from 'crypto'
 import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
 import * as jwt from 'jsonwebtoken'
+import { parseHTML } from 'linkedom'
 import Parser, { Item } from 'rss-parser'
 import { promisify } from 'util'
 import { CONTENT_FETCH_URL, createCloudTask } from './task'
@@ -87,6 +88,48 @@ export const fetchAndChecksum = async (url: string) => {
   }
 }
 
+const parseFeed = async (url: string, content: string) => {
+  try {
+    // check if url is a telegram channel
+    const telegramRegex = /https:\/\/t\.me\/([a-zA-Z0-9_]+)/
+    const telegramMatch = url.match(telegramRegex)
+    if (telegramMatch) {
+      const dom = parseHTML(content).document
+      const title = dom.querySelector('meta[property="og:title"]')
+      // post has attribute data-post
+      const posts = dom.querySelectorAll('[data-post]')
+      const items = Array.from(posts)
+        .map((post) => {
+          const id = post.getAttribute('data-post')
+          if (!id) {
+            return null
+          }
+
+          const url = `https://t.me/${telegramMatch[1]}/${id}`
+          // find the <time> element
+          const time = post.querySelector('time')
+          const dateTime = time?.getAttribute('datetime') || undefined
+
+          return {
+            link: url,
+            isoDate: dateTime,
+          }
+        })
+        .filter((item) => !!item) as RssFeedItem[]
+
+      return {
+        title: title?.getAttribute('content') || dom.title,
+        items,
+      }
+    }
+
+    return parser.parseString(content)
+  } catch (error) {
+    console.log(error)
+    return null
+  }
+}
+
 const sendUpdateSubscriptionMutation = async (
   userId: string,
   subscriptionId: string,
@@ -477,7 +520,12 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
       console.log('Processing feed', feedUrl)
 
       const fetchResult = await fetchAndChecksum(feedUrl)
-      const feed = await parser.parseString(fetchResult.content)
+      const feed = await parseFeed(feedUrl, fetchResult.content)
+      if (!feed) {
+        console.error('Failed to parse RSS feed', feedUrl)
+        return res.status(500).send('INVALID_RSS_FEED')
+      }
+
       console.log('Fetched feed', feed.title, new Date())
 
       await Promise.all(
@@ -498,7 +546,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction(
 
       res.send('ok')
     } catch (e) {
-      console.error('Error while parsing RSS feed', e)
+      console.error('Error while saving RSS feeds', e)
       res.status(500).send('INTERNAL_SERVER_ERROR')
     }
   }

yarn.lock

@@ -12581,7 +12581,7 @@ domhandler@^4.3.1:
   dependencies:
     domelementtype "^2.2.0"
 
-domhandler@^5.0.1, domhandler@^5.0.2:
+domhandler@^5.0.1, domhandler@^5.0.2, domhandler@^5.0.3:
   version "5.0.3"
   resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31"
   integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==
@@ -12630,6 +12630,15 @@ domutils@^3.0.1:
     domelementtype "^2.3.0"
     domhandler "^5.0.1"
 
+domutils@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.1.0.tgz#c47f551278d3dc4b0b1ab8cbb42d751a6f0d824e"
+  integrity sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==
+  dependencies:
+    dom-serializer "^2.0.0"
+    domelementtype "^2.3.0"
+    domhandler "^5.0.3"
+
 dot-case@^2.1.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/dot-case/-/dot-case-2.1.1.tgz#34dcf37f50a8e93c2b3bca8bb7fb9155c7da3bee"
@@ -12943,6 +12952,11 @@ entities@^4.2.0, entities@^4.3.0:
   resolved "https://registry.yarnpkg.com/entities/-/entities-4.3.0.tgz#62915f08d67353bb4eb67e3d62641a4059aec656"
   integrity sha512-/iP1rZrSEJ0DTlPiX+jbzlA3eVkY/e8L8SozroF395fIqE3TYF/Nz7YOMAawta+vLmyJ/hkGNNPcSbMADCCXbg==
 
+entities@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48"
+  integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==
+
 entities@~2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5"
@@ -16145,6 +16159,16 @@ htmlparser2@^8.0.1:
     domutils "^3.0.1"
     entities "^4.3.0"
 
+htmlparser2@^9.0.0:
+  version "9.0.0"
+  resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-9.0.0.tgz#e431142b7eeb1d91672742dea48af8ac7140cddb"
+  integrity sha512-uxbSI98wmFT/G4P2zXx4OVx04qWUmyFPrD2/CNepa2Zo3GPNaCaaxElDgwUrwYWkK1nr9fft0Ya8dws8coDLLQ==
+  dependencies:
+    domelementtype "^2.3.0"
+    domhandler "^5.0.3"
+    domutils "^3.1.0"
+    entities "^4.5.0"
+
 htmltidy2@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/htmltidy2/-/htmltidy2-0.3.0.tgz#1edfb74b8cd530cdcdc29ef547c849a651f0870b"
@@ -18755,6 +18779,17 @@ linkedom@^0.14.9:
     htmlparser2 "^8.0.1"
     uhyphen "^0.1.0"
 
+linkedom@^0.16.4:
+  version "0.16.4"
+  resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.16.4.tgz#6ea2711d03196b58af01fa8acab26cb231f38baf"
+  integrity sha512-SykvDVh/jAnaO+WiPqH5vX3QpZrIRImuppzYhIHons3RXPhDwqN2dOyfopOVaHleqWtoS+3vWCqen+m8M3HToQ==
+  dependencies:
+    css-select "^5.1.0"
+    cssom "^0.5.0"
+    html-escaper "^3.0.3"
+    htmlparser2 "^9.0.0"
+    uhyphen "^0.2.0"
+
 linkify-it@^3.0.1:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-3.0.3.tgz#a98baf44ce45a550efb4d49c769d07524cc2fa2e"