allow adding telegram channel as feed

packages/api/src/resolvers/article/index.ts

@@ -21,8 +21,12 @@ import {
   CreateArticleError,
   CreateArticleErrorCode,
   CreateArticleSuccess,
+  MoveToFolderError,
+  MoveToFolderErrorCode,
+  MoveToFolderSuccess,
   MutationBulkActionArgs,
   MutationCreateArticleArgs,
+  MutationMoveToFolderArgs,
   MutationSaveArticleReadingProgressArgs,
   MutationSetBookmarkArticleArgs,
   MutationSetFavoriteArticleArgs,
@@ -85,6 +89,7 @@ import {
   generateSlug,
   isParsingTimeout,
   libraryItemToArticle,
+  libraryItemToArticleSavingRequest,
   libraryItemToSearchItem,
   titleForFilePath,
   userDataToUser,
@@ -870,6 +875,80 @@ export const setFavoriteArticleResolver = authorized<
   }
 })
 
+export const moveToFolderResolver = authorized<
+  MoveToFolderSuccess,
+  MoveToFolderError,
+  MutationMoveToFolderArgs
+>(async (_, { id, folder }, { authTrx, pubsub, uid }) => {
+  analytics.track({
+    userId: uid,
+    event: 'move_to_folder',
+    properties: {
+      id,
+      folder,
+    },
+  })
+
+  const item = await authTrx((tx) =>
+    tx.getRepository(LibraryItem).findOne({
+      where: {
+        id,
+      },
+      relations: ['user'],
+    })
+  )
+
+  if (!item) {
+    return {
+      errorCodes: [MoveToFolderErrorCode.Unauthorized],
+    }
+  }
+
+  if (item.folder === folder) {
+    return {
+      errorCodes: [MoveToFolderErrorCode.AlreadyExists],
+    }
+  }
+
+  const savedAt = new Date()
+
+  // if the content is not fetched yet, create a page save request
+  if (!item.readableContent) {
+    const articleSavingRequest = await createPageSaveRequest({
+      userId: uid,
+      url: item.originalUrl,
+      articleSavingRequestId: id,
+      priority: 'high',
+      publishedAt: item.publishedAt || undefined,
+      savedAt,
+      pubsub,
+    })
+
+    return {
+      __typename: 'MoveToFolderSuccess',
+      articleSavingRequest,
+    }
+  }
+
+  const updatedItem = await updateLibraryItem(
+    item.id,
+    {
+      folder,
+      savedAt,
+    },
+    uid,
+    pubsub
+  )
+
+  return {
+    __typename: 'MoveToFolderSuccess',
+    articleSavingRequest: libraryItemToArticleSavingRequest(
+      updatedItem.user,
+      updatedItem
+    ),
+  }
+})
+
 const getUpdateReason = (libraryItem: LibraryItem, since: Date) => {
   if (libraryItem.deletedAt) {
     return UpdateReason.Deleted

packages/api/src/resolvers/following/index.ts

@@ -1,231 +0,0 @@
-import axios from 'axios'
-import { parseHTML } from 'linkedom'
-import { LibraryItem } from '../../entity/library_item'
-import {
-  FeedEdge,
-  FeedsError,
-  FeedsErrorCode,
-  FeedsSuccess,
-  MoveToFolderError,
-  MoveToFolderErrorCode,
-  MoveToFolderSuccess,
-  MutationMoveToFolderArgs,
-  QueryFeedsArgs,
-  QueryScanFeedsArgs,
-  ScanFeedsError,
-  ScanFeedsErrorCode,
-  ScanFeedsSuccess,
-  ScanFeedsType,
-} from '../../generated/graphql'
-import { feedRepository } from '../../repository/feed'
-import { createPageSaveRequest } from '../../services/create_page_save_request'
-import { updateLibraryItem } from '../../services/library_item'
-import { analytics } from '../../utils/analytics'
-import {
-  authorized,
-  libraryItemToArticleSavingRequest,
-} from '../../utils/helpers'
-import { parseOpml } from '../../utils/parser'
-
-export const feedsResolver = authorized<
-  FeedsSuccess,
-  FeedsError,
-  QueryFeedsArgs
->(async (_, { input }, { log }) => {
-  try {
-    const startCursor = input.after || ''
-    const start =
-      startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0
-    const first = Math.min(input.first || 10, 100) // cap at 100
-
-    const { feeds, count } = await feedRepository.searchFeeds(
-      input.query || '',
-      first + 1, // fetch one extra to check if there is a next page
-      start,
-      input.sort?.by,
-      input.sort?.order || undefined
-    )
-
-    const hasNextPage = feeds.length > first
-    const endCursor = String(start + feeds.length - (hasNextPage ? 1 : 0))
-
-    if (hasNextPage) {
-      // remove an extra if exists
-      feeds.pop()
-    }
-
-    const edges: FeedEdge[] = feeds.map((feed) => ({
-      node: feed,
-      cursor: endCursor,
-    }))
-
-    return {
-      __typename: 'FeedsSuccess',
-      edges,
-      pageInfo: {
-        hasPreviousPage: start > 0,
-        hasNextPage,
-        startCursor,
-        endCursor,
-        totalCount: count,
-      },
-    }
-  } catch (error) {
-    log.error('Error fetching feeds', error)
-
-    return {
-      errorCodes: [FeedsErrorCode.BadRequest],
-    }
-  }
-})
-
-export const moveToFolderResolver = authorized<
-  MoveToFolderSuccess,
-  MoveToFolderError,
-  MutationMoveToFolderArgs
->(async (_, { id, folder }, { authTrx, pubsub, uid }) => {
-  analytics.track({
-    userId: uid,
-    event: 'move_to_folder',
-    properties: {
-      id,
-      folder,
-    },
-  })
-
-  const item = await authTrx((tx) =>
-    tx.getRepository(LibraryItem).findOne({
-      where: {
-        id,
-      },
-      relations: ['user'],
-    })
-  )
-
-  if (!item) {
-    return {
-      errorCodes: [MoveToFolderErrorCode.Unauthorized],
-    }
-  }
-
-  if (item.folder === folder) {
-    return {
-      errorCodes: [MoveToFolderErrorCode.AlreadyExists],
-    }
-  }
-
-  const savedAt = new Date()
-
-  // if the content is not fetched yet, create a page save request
-  if (!item.readableContent) {
-    const articleSavingRequest = await createPageSaveRequest({
-      userId: uid,
-      url: item.originalUrl,
-      articleSavingRequestId: id,
-      priority: 'high',
-      publishedAt: item.publishedAt || undefined,
-      savedAt,
-      pubsub,
-    })
-
-    return {
-      __typename: 'MoveToFolderSuccess',
-      articleSavingRequest,
-    }
-  }
-
-  const updatedItem = await updateLibraryItem(
-    item.id,
-    {
-      folder,
-      savedAt,
-    },
-    uid,
-    pubsub
-  )
-
-  return {
-    __typename: 'MoveToFolderSuccess',
-    articleSavingRequest: libraryItemToArticleSavingRequest(
-      updatedItem.user,
-      updatedItem
-    ),
-  }
-})
-
-export const scanFeedsResolver = authorized<
-  ScanFeedsSuccess,
-  ScanFeedsError,
-  QueryScanFeedsArgs
->(async (_, { input: { type, opml, url } }, { log, uid }) => {
-  analytics.track({
-    userId: uid,
-    event: 'scan_feeds',
-    properties: {
-      type,
-    },
-  })
-
-  if (type === ScanFeedsType.Opml) {
-    if (!opml) {
-      return {
-        errorCodes: [ScanFeedsErrorCode.BadRequest],
-      }
-    }
-
-    // parse opml
-    const feeds = parseOpml(opml)
-    if (!feeds) {
-      return {
-        errorCodes: [ScanFeedsErrorCode.BadRequest],
-      }
-    }
-
-    return {
-      __typename: 'ScanFeedsSuccess',
-      feeds: feeds.map((feed) => ({
-        url: feed.feedUrl,
-        title: feed.title,
-        type: feed.feedType || 'rss',
-      })),
-    }
-  }
-
-  if (!url) {
-    return {
-      errorCodes: [ScanFeedsErrorCode.BadRequest],
-    }
-  }
-
-  try {
-    // fetch HTML and parse feeds
-    const response = await axios.get(url, {
-      timeout: 5000,
-      headers: {
-        'User-Agent': 'Mozilla/5.0',
-        Accept: 'text/html',
-      },
-    })
-    const html = response.data as string
-    const dom = parseHTML(html).document
-    const links = dom.querySelectorAll('link[type="application/rss+xml"]')
-    const feeds = Array.from(links)
-      .map((link) => ({
-        url: link.getAttribute('href') || '',
-        title: link.getAttribute('title') || '',
-        type: 'rss',
-      }))
-      .filter((feed) => feed.url)
-
-    return {
-      __typename: 'ScanFeedsSuccess',
-      feeds,
-    }
-  } catch (error) {
-    log.error('Error scanning HTML', error)
-
-    return {
-      errorCodes: [ScanFeedsErrorCode.BadRequest],
-    }
-  }
-})

packages/api/src/resolvers/function_resolvers.ts

@@ -29,7 +29,6 @@ import {
   generateUploadFilePathName,
 } from '../utils/uploads'
 import { optInFeatureResolver } from './features'
-import { feedsResolver, moveToFolderResolver } from './following'
 import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
 import {
   addPopularReadResolver,
@@ -53,6 +52,7 @@ import {
   deleteRuleResolver,
   deleteWebhookResolver,
   deviceTokensResolver,
+  feedsResolver,
   filtersResolver,
   generateApiKeyResolver,
   getAllUsersResolver,
@@ -76,6 +76,7 @@ import {
   mergeHighlightResolver,
   moveFilterResolver,
   moveLabelResolver,
+  moveToFolderResolver,
   newsletterEmailsResolver,
   recommendHighlightsResolver,
   recommendResolver,
@@ -88,6 +89,7 @@ import {
   saveFilterResolver,
   savePageResolver,
   saveUrlResolver,
+  scanFeedsResolver,
   searchResolver,
   sendInstallInstructionsResolver,
   setBookmarkArticleResolver,
@@ -249,6 +251,7 @@ export const functionResolvers = {
     groups: groupsResolver,
     recentEmails: recentEmailsResolver,
     feeds: feedsResolver,
+    scanFeeds: scanFeedsResolver,
   },
   User: {
     async intercomHash(

packages/api/src/resolvers/subscriptions/index.ts

@@ -1,12 +1,24 @@
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
 import Parser from 'rss-parser'
 import { Brackets } from 'typeorm'
 import { Subscription } from '../../entity/subscription'
 import { env } from '../../env'
 import {
+  FeedEdge,
+  FeedsError,
+  FeedsErrorCode,
+  FeedsSuccess,
   MutationSubscribeArgs,
   MutationUnsubscribeArgs,
   MutationUpdateSubscriptionArgs,
+  QueryFeedsArgs,
+  QueryScanFeedsArgs,
   QuerySubscriptionsArgs,
+  ScanFeedsError,
+  ScanFeedsErrorCode,
+  ScanFeedsSuccess,
+  ScanFeedsType,
   SortBy,
   SortOrder,
   SubscribeError,
@@ -25,11 +37,13 @@ import {
   UpdateSubscriptionSuccess,
 } from '../../generated/graphql'
 import { getRepository } from '../../repository'
+import { feedRepository } from '../../repository/feed'
 import { unsubscribe } from '../../services/subscriptions'
 import { Merge } from '../../util'
 import { analytics } from '../../utils/analytics'
 import { enqueueRssFeedFetch } from '../../utils/createTask'
 import { authorized } from '../../utils/helpers'
+import { parseFeed, parseOpml } from '../../utils/parser'
 
 type PartialSubscription = Omit<Subscription, 'newsletterEmail'>
 
@@ -175,7 +189,7 @@ export const subscribeResolver = authorized<
   SubscribeSuccessPartial,
   SubscribeError,
   MutationSubscribeArgs
->(async (_, { input }, { authTrx, uid, log }) => {
+>(async (_, { input }, { uid, log }) => {
   try {
     analytics.track({
       userId: uid,
@@ -224,7 +238,12 @@ export const subscribeResolver = authorized<
     // create new rss subscription
     const MAX_RSS_SUBSCRIPTIONS = 150
     // validate rss feed
-    const feed = await parser.parseURL(input.url)
+    const feed = await parseFeed(input.url)
+    if (!feed) {
+      return {
+        errorCodes: [SubscribeErrorCode.NotFound],
+      }
+    }
 
     // limit number of rss subscriptions to 150
     const results = (await getRepository(Subscription).query(
@@ -235,11 +254,11 @@ export const subscribeResolver = authorized<
           returning *;`,
       [
         feed.title,
-        feed.feedUrl,
+        feed.url,
         feed.description || null,
         SubscriptionType.Rss,
         uid,
-        feed.image?.url || null,
+        feed.thumbnail || null,
         input.autoAddToLibrary ?? null,
         input.isPrivate ?? null,
         MAX_RSS_SUBSCRIPTIONS,
@@ -336,3 +355,132 @@ export const updateSubscriptionResolver = authorized<
     }
   }
 })
+
+export const feedsResolver = authorized<
+  FeedsSuccess,
+  FeedsError,
+  QueryFeedsArgs
+>(async (_, { input }, { log }) => {
+  try {
+    const startCursor = input.after || ''
+    const start =
+      startCursor && !isNaN(Number(startCursor)) ? Number(startCursor) : 0
+    const first = Math.min(input.first || 10, 100) // cap at 100
+
+    const { feeds, count } = await feedRepository.searchFeeds(
+      input.query || '',
+      first + 1, // fetch one extra to check if there is a next page
+      start,
+      input.sort?.by,
+      input.sort?.order || undefined
+    )
+
+    const hasNextPage = feeds.length > first
+    const endCursor = String(start + feeds.length - (hasNextPage ? 1 : 0))
+
+    if (hasNextPage) {
+      // remove an extra if exists
+      feeds.pop()
+    }
+
+    const edges: FeedEdge[] = feeds.map((feed) => ({
+      node: feed,
+      cursor: endCursor,
+    }))
+
+    return {
+      __typename: 'FeedsSuccess',
+      edges,
+      pageInfo: {
+        hasPreviousPage: start > 0,
+        hasNextPage,
+        startCursor,
+        endCursor,
+        totalCount: count,
+      },
+    }
+  } catch (error) {
+    log.error('Error fetching feeds', error)
+
+    return {
+      errorCodes: [FeedsErrorCode.BadRequest],
+    }
+  }
+})
+
+export const scanFeedsResolver = authorized<
+  ScanFeedsSuccess,
+  ScanFeedsError,
+  QueryScanFeedsArgs
+>(async (_, { input: { type, opml, url } }, { log, uid }) => {
+  analytics.track({
+    userId: uid,
+    event: 'scan_feeds',
+    properties: {
+      type,
+    },
+  })
+
+  if (type === ScanFeedsType.Opml) {
+    if (!opml) {
+      return {
+        errorCodes: [ScanFeedsErrorCode.BadRequest],
+      }
+    }
+
+    // parse opml
+    const feeds = parseOpml(opml)
+    if (!feeds) {
+      return {
+        errorCodes: [ScanFeedsErrorCode.BadRequest],
+      }
+    }
+
+    return {
+      __typename: 'ScanFeedsSuccess',
+      feeds: feeds.map((feed) => ({
+        url: feed.url,
+        title: feed.title,
+        type: feed.type || 'rss',
+      })),
+    }
+  }
+
+  if (!url) {
+    return {
+      errorCodes: [ScanFeedsErrorCode.BadRequest],
+    }
+  }
+
+  try {
+    // fetch HTML and parse feeds
+    const response = await axios.get(url, {
+      timeout: 5000,
+      headers: {
+        'User-Agent': 'Mozilla/5.0',
+        Accept: 'text/html',
+      },
+    })
+    const html = response.data as string
+    const dom = parseHTML(html).document
+    const links = dom.querySelectorAll('link[type="application/rss+xml"]')
+    const feeds = Array.from(links)
+      .map((link) => ({
+        url: link.getAttribute('href') || '',
+        title: link.getAttribute('title') || '',
+        type: 'rss',
+      }))
+      .filter((feed) => feed.url)
+
+    return {
+      __typename: 'ScanFeedsSuccess',
+      feeds,
+    }
+  } catch (error) {
+    log.error('Error scanning HTML', error)
+
+    return {
+      errorCodes: [ScanFeedsErrorCode.BadRequest],
+    }
+  }
+})

packages/api/src/utils/parser.ts

@@ -12,6 +12,7 @@ import * as jwt from 'jsonwebtoken'
 import { parseHTML } from 'linkedom'
 import { NodeHtmlMarkdown, TranslatorConfigObject } from 'node-html-markdown'
 import { ElementNode } from 'node-html-markdown/dist/nodes'
+import Parser from 'rss-parser'
 import { parser } from 'sax'
 import { ILike } from 'typeorm'
 import { promisify } from 'util'
@@ -35,13 +36,23 @@ import { buildLogger, LogRecord } from './logger'
 interface Feed {
   title: string
   url: string
-  feedUrl: string
-  feedType: string
+  type: string
+  thumbnail?: string
+  description?: string
 }
 
 const logger = buildLogger('utils.parse')
 const signToken = promisify(jwt.sign)
 
+const axiosInstance = axios.create({
+  timeout: 5000,
+  headers: {
+    'User-Agent': 'Mozilla/5.0',
+    Accept: 'text/html',
+  },
+  responseType: 'text',
+})
+
 export const ALLOWED_CONTENT_TYPES = [
   'text/html',
   'application/octet-stream',
@@ -712,6 +723,16 @@ export const getDistillerResult = async (
   }
 }
 
+const fetchHtml = async (url: string): Promise<string | undefined> => {
+  try {
+    const response = await axiosInstance.get(url)
+    return response.data as string
+  } catch (error) {
+    logger.error('Error fetching html', error)
+    return undefined
+  }
+}
+
 export const parseOpml = (opml: string): Feed[] | undefined => {
   const xmlParser = parser(true, { lowercase: true })
   const feeds: Feed[] = []
@@ -723,13 +744,9 @@ export const parseOpml = (opml: string): Feed[] | undefined => {
       const feedUrl = node.attributes.xmlUrl.toString()
       if (feedUrl && !existingFeeds.has(feedUrl)) {
         feeds.push({
-          title:
-            node.attributes.title.toString() ||
-            node.attributes.text.toString() ||
-            node.attributes.description.toString(),
-          url: node.attributes.htmlUrl.toString(),
-          feedUrl: feedUrl.toString(),
-          feedType: node.attributes.type.toString(),
+          title: node.attributes.title.toString() || '',
+          url: feedUrl,
+          type: node.attributes.type.toString() || 'rss',
         })
         existingFeeds.set(feedUrl, true)
       }
@@ -747,3 +764,77 @@ export const parseOpml = (opml: string): Feed[] | undefined => {
     return undefined
   }
 }
+
+export const parseHtml = async (url: string): Promise<Feed[] | undefined> => {
+  // fetch HTML and parse feeds
+  const html = await fetchHtml(url)
+  if (!html) return undefined
+
+  try {
+    const dom = parseHTML(html).document
+    const links = dom.querySelectorAll('link[type="application/rss+xml"]')
+    const feeds = Array.from(links)
+      .map((link) => ({
+        url: link.getAttribute('href') || '',
+        title: link.getAttribute('title') || '',
+        type: 'rss',
+      }))
+      .filter((feed) => feed.url)
+
+    return feeds
+  } catch (error) {
+    logger.error('Error parsing html', error)
+    return undefined
+  }
+}
+
+export const parseFeed = async (url: string): Promise<Feed | undefined> => {
+  try {
+    // check if url is a telegram channel
+    const telegramRegex = /https:\/\/t\.me\/([a-zA-Z0-9_]+)/
+    const telegramMatch = url.match(telegramRegex)
+    if (telegramMatch) {
+      // fetch HTML and parse feeds
+      const html = await fetchHtml(url)
+      if (!html) return undefined
+
+      const dom = parseHTML(html).document
+      const title = dom.querySelector('meta[property="og:title"]')
+      const thumbnail = dom.querySelector('meta[property="og:image"]')
+      const description = dom.querySelector('meta[property="og:description"]')
+
+      return {
+        title: title?.getAttribute('content') || url,
+        url,
+        type: 'telegram',
+        thumbnail: thumbnail?.getAttribute('content') || '',
+        description: description?.getAttribute('content') || '',
+      }
+    }
+
+    const parser = new Parser({
+      timeout: 5000, // 5 seconds
+      headers: {
+        // some rss feeds require user agent
+        'User-Agent':
+          'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
+        Accept:
+          'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4',
+      },
+    })
+
+    const feed = await parser.parseURL(url)
+    const feedUrl = feed.feedUrl || url
+
+    return {
+      title: feed.title || feedUrl,
+      url: feedUrl,
+      thumbnail: feed.image?.url,
+      type: 'rss',
+      description: feed.description,
+    }
+  } catch (error) {
+    logger.error('Error parsing feed', error)
+    return undefined
+  }
+}