Add handlers for content-fetching

packages/content-handler/package.json

@@ -16,5 +16,11 @@
   },
   "devDependencies": {
     "eslint-plugin-prettier": "^4.0.0"
+  },
+  "dependencies": {
+    "axios": "^0.27.2",
+    "linkedom": "^0.14.16",
+    "luxon": "^3.0.4",
+    "underscore": "^1.13.6"
   }
 }

packages/content-handler/src/handlers/apple-news-handler.ts

@@ -0,0 +1,26 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+export class AppleNewsHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    return u.hostname === 'apple.news'
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    const MOBILE_USER_AGENT =
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
+    const response = await axios.get(url, {
+      headers: { 'User-Agent': MOBILE_USER_AGENT },
+    })
+    const data = response.data
+    const dom = parseHTML(data).document
+    // make sure it's a valid URL by wrapping in new URL
+    const href = dom
+      .querySelector('span.click-here')
+      ?.parentElement?.getAttribute('href')
+    const u = href ? new URL(href) : undefined
+    return { url: u?.href }
+  }
+}

packages/content-handler/src/handlers/bloomberg-handler.ts

@@ -0,0 +1,36 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+class BloombergHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const BLOOMBERG_URL_MATCH =
+      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
+    return BLOOMBERG_URL_MATCH.test(url.toString())
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    console.log('prehandling bloomberg url', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          api_key: process.env.SCRAPINGBEE_API_KEY,
+          url: url,
+          return_page_source: true,
+          block_ads: true,
+          block_resources: false,
+        },
+      })
+      const dom = parseHTML(response.data).document
+      return {
+        title: dom.title,
+        content: dom.querySelector('body')?.innerHTML,
+        url: url,
+      }
+    } catch (error) {
+      console.error('error prehandling bloomberg url', error)
+      throw error
+    }
+  }
+}

packages/content-handler/src/handlers/derstandard-handler.ts

@@ -0,0 +1,29 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+class DerstandardHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    return u.hostname === 'www.derstandard.at'
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    const response = await axios.get(url, {
+      // set cookie to give consent to get the article
+      headers: {
+        cookie: `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`,
+      },
+    })
+    const content = response.data
+
+    const dom = parseHTML(content).document
+    const titleElement = dom.querySelector('.article-title')
+    titleElement && titleElement.remove()
+
+    return {
+      content: dom.body.outerHTML,
+      title: titleElement?.textContent || undefined,
+    }
+  }
+}

packages/content-handler/src/handlers/image-handler.ts

@@ -0,0 +1,27 @@
+import { ContentHandler, PreHandleResult } from '../index'
+
+class ImageHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
+    return IMAGE_URL_PATTERN.test(url.toString())
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    const title = url.toString().split('/').pop()
+    const content = `
+      <html>
+        <head>
+          <title>${title}</title>
+          <meta property="og:image" content="${url}" />
+          <meta property="og:title" content="${title}" />
+        </head>
+        <body>
+          <div>
+            <img src="${url}" alt="${title}">
+          </div>
+        </body>
+      </html>`
+
+    return { title, content }
+  }
+}

packages/content-handler/src/handlers/medium-handler.ts

@@ -0,0 +1,21 @@
+import { ContentHandler, PreHandleResult } from '../index'
+
+class MediumHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    return u.hostname.endsWith('medium.com')
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    console.log('prehandling medium url', url)
+
+    try {
+      const res = new URL(url)
+      res.searchParams.delete('source')
+      return { url: res.toString() }
+    } catch (error) {
+      console.error('error prehandling medium url', error)
+      throw error
+    }
+  }
+}

packages/content-handler/src/handlers/pdf-handler.ts

@@ -0,0 +1,13 @@
+import { ContentHandler, PreHandleResult } from '../index'
+
+class PdfHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    const path = u.pathname.replace(u.search, '')
+    return path.endsWith('.pdf')
+  }
+
+  async preHandle(_url: string, _document: Document): Promise<PreHandleResult> {
+    return { contentType: 'application/pdf' }
+  }
+}

packages/content-handler/src/handlers/scrapingBee-handler.ts

@@ -0,0 +1,33 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { parseHTML } from 'linkedom'
+
+class ScrapingBeeHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    const u = new URL(url)
+    const hostnames = ['nytimes.com', 'news.google.com']
+
+    return hostnames.some((h) => u.hostname.endsWith(h))
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    console.log('prehandling url with scrapingbee', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          api_key: process.env.SCRAPINGBEE_API_KEY,
+          url: url,
+          return_page_source: true,
+          block_ads: true,
+          block_resources: false,
+        },
+      })
+      const dom = parseHTML(response.data).document
+      return { title: dom.title, content: response.data, url: url }
+    } catch (error) {
+      console.error('error prehandling url w/scrapingbee', error)
+      throw error
+    }
+  }
+}

packages/content-handler/src/handlers/t-dot-co-handler.ts

@@ -0,0 +1,21 @@
+import { ContentHandler } from '../index'
+import axios from 'axios'
+
+class TDotCoHandler extends ContentHandler {
+  shouldResolve(url: string): boolean {
+    const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/
+    return T_DOT_CO_URL_MATCH.test(url)
+  }
+
+  async resolve(url: string) {
+    return await axios
+      .get(url, { maxRedirects: 0, validateStatus: null })
+      .then((res) => {
+        return new URL(res.headers.location).href
+      })
+      .catch((err) => {
+        console.log('err with t.co url', err)
+        return undefined
+      })
+  }
+}

packages/content-handler/src/handlers/twitter-handler.ts

@@ -0,0 +1,142 @@
+import { ContentHandler, PreHandleResult } from '../index'
+import axios from 'axios'
+import { DateTime } from 'luxon'
+import _ from 'underscore'
+
+const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
+const TWITTER_URL_MATCH =
+  /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
+
+const getTweetFields = () => {
+  const TWEET_FIELDS =
+    '&tweet.fields=attachments,author_id,conversation_id,created_at,' +
+    'entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets,' +
+    'source,withheld'
+  const EXPANSIONS = '&expansions=author_id,attachments.media_keys'
+  const USER_FIELDS =
+    '&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld'
+  const MEDIA_FIELDS =
+    '&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width'
+
+  return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`
+}
+
+const getTweetById = async (id: string) => {
+  const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
+  const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
+
+  return await axios.get(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: 'follow',
+    },
+  })
+}
+
+const titleForAuthor = (author: any) => {
+  return `${author.name} on Twitter`
+}
+
+const tweetIdFromStatusUrl = (url: string): string | undefined => {
+  const match = url.toString().match(TWITTER_URL_MATCH)
+  return match?.[2]
+}
+
+const formatTimestamp = (timestamp: string) => {
+  return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(
+    DateTime.DATETIME_FULL
+  )
+}
+
+class TwitterHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
+  }
+
+  // version of the handler that uses the oembed API
+  // This isn't great as it doesn't work well with our
+  // readability API. But could potentially give a more consistent
+  // look to the tweets
+  // prehandle: async (url, env) => {
+  //   const oeTweet = await embeddedTweet(url)
+  //   const dom = new JSDOM(oeTweet.data.html);
+  //   const bq = dom.window.document.querySelector('blockquote')
+  //   console.log('blockquote:', bq);
+
+  //   const title = titleForTweet(oeTweet)
+  //   return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
+  // }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    console.log('prehandling twitter url', url)
+
+    const tweetId = tweetIdFromStatusUrl(url)
+    if (!tweetId) {
+      throw new Error('could not find tweet id in url')
+    }
+    const tweetData = (await getTweetById(tweetId)).data
+    const authorId = tweetData.data.author_id
+    const author = tweetData.includes.users.filter(
+      (u: any) => (u.id = authorId)
+    )[0]
+    // escape html entities in title
+    const title = _.escape(titleForAuthor(author))
+    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
+
+    let text = tweetData.data.text
+    if (tweetData.data.entities && tweetData.data.entities.urls) {
+      for (let urlObj of tweetData.data.entities.urls) {
+        text = text.replace(
+          urlObj.url,
+          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
+        )
+      }
+    }
+
+    const front = `
+    <div>
+      <p>${text}</p>
+    `
+
+    var includesHtml = ''
+    if (tweetData.includes.media) {
+      includesHtml = tweetData.includes.media
+        .map((m: any) => {
+          const linkUrl = m.type == 'photo' ? m.url : url
+          const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
+          const mediaOpen = `<a class="media-link" href=${linkUrl}>
+          <picture>
+            <img class="tweet-img" src=${previewUrl} />
+          </picture>
+          </a>`
+          return mediaOpen
+        })
+        .join('\n')
+    }
+
+    const back = `
+       — <a href="https://twitter.com/${author.username}">${
+      author.username
+    }</a> ${author.name} <a href="${url}">${formatTimestamp(
+      tweetData.data.created_at
+    )}</a>
+    </div>
+    `
+    const content = `
+    <head>
+      <meta property="og:image" content="${authorImage}" />
+      <meta property="og:image:secure_url" content="${authorImage}" />
+      <meta property="og:title" content="${title}" />
+      <meta property="og:description" content="${_.escape(
+        tweetData.data.text
+      )}" />
+    </head>
+    <body>
+      ${front}
+      ${includesHtml}
+      ${back}
+    </body>`
+
+    return { content, url, title }
+  }
+}

packages/content-handler/src/handlers/youtube-handler.ts

@@ -0,0 +1,66 @@
+import { PreHandleResult } from '../index'
+import axios from 'axios'
+import _ from 'underscore'
+
+const { ContentHandler } = require('../index')
+
+const YOUTUBE_URL_MATCH =
+  /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
+
+function getYoutubeVideoId(url: string) {
+  const u = new URL(url)
+  const videoId = u.searchParams.get('v')
+  if (!videoId) {
+    const match = url.toString().match(YOUTUBE_URL_MATCH)
+    if (match === null || match.length < 6 || !match[5]) {
+      return undefined
+    }
+    return match[5]
+  }
+  return videoId
+}
+
+class YoutubeHandler extends ContentHandler {
+  shouldPreHandle(url: string, _dom: Document): boolean {
+    return YOUTUBE_URL_MATCH.test(url.toString())
+  }
+
+  async preHandle(url: string, _document: Document): Promise<PreHandleResult> {
+    const videoId = getYoutubeVideoId(url)
+    if (!videoId) {
+      return {}
+    }
+
+    const oembedUrl =
+      `https://www.youtube.com/oembed?format=json&url=` +
+      encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
+    const oembed = (await axios.get(oembedUrl.toString())).data
+    // escape html entities in title
+    const title = _.escape(oembed.title)
+    const ratio = oembed.width / oembed.height
+    const thumbnail = oembed.thumbnail_url
+    const height = 350
+    const width = height * ratio
+    const authorName = _.escape(oembed.author_name)
+
+    const content = `
+    <html>
+      <head><title>${title}</title>
+      <meta property="og:image" content="${thumbnail}" />
+      <meta property="og:image:secure_url" content="${thumbnail}" />
+      <meta property="og:title" content="${title}" />
+      <meta property="og:description" content="" />
+      <meta property="og:article:author" content="${authorName}" />
+      </head>
+      <body>
+      <iframe width="${width}" height="${height}" src="https://www.youtube.com/embed/${videoId}" title="${title}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+        <p><a href="${url}" target="_blank">${title}</a></p>
+        <p itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}" target="_blank">${authorName}</a></p>
+      </body>
+    </html>`
+
+    console.log('got video id', videoId)
+
+    return { content, title: 'Youtube Content' }
+  }
+}

packages/content-handler/src/index.ts

@@ -17,18 +17,26 @@ interface NewsletterMessage {
   unsubHttpUrl?: string
 }
 
+export interface PreHandleResult {
+  url?: string
+  title?: string
+  content?: string
+  contentType?: string
+  dom?: Document
+}
+
 export class ContentHandler {
   protected senderRegex = /NEWSLETTER_SENDER_REGEX/
   protected urlRegex = /NEWSLETTER_URL_REGEX/
   protected defaultUrl = 'NEWSLETTER_DEFAULT_URL'
   protected name = ''
 
-  shouldPrehandle(url: URL, dom: Document): boolean {
+  shouldPreHandle(url: string, dom: Document): boolean {
     return false
   }
 
-  prehandle(url: URL, document: Document): Promise<Document> {
-    return Promise.resolve(document)
+  preHandle(url: string, document: Document): Promise<PreHandleResult> {
+    return Promise.resolve({ url, dom: document })
   }
 
   isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean {
@@ -67,14 +75,14 @@ export class ContentHandler {
     }
   }
 
-  async handleNewsletter(
+  handleNewsletter(
     email: string,
     html: string,
     postHeader: string,
     title: string,
     from: string,
     unSubHeader: string
-  ): Promise<NewsletterMessage> {
+  ): NewsletterMessage {
     console.log('handleNewsletter', email, postHeader, title, from)
 
     if (!email || !html || !title || !from) {

yarn.lock

@@ -17652,6 +17652,17 @@ linkedom@^0.14.12:
     htmlparser2 "^8.0.1"
     uhyphen "^0.1.0"
 
+linkedom@^0.14.16:
+  version "0.14.16"
+  resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.16.tgz#124eb006fad1dfe7ed8f96ec8ae74ab0fb0fd88e"
+  integrity sha512-a4QWl4W93P15/x+4d9k8K+C81nOzQeGOs3D37uG0TFqKZYGLEyZwXweSFrypK8yvUx5U2cuZKkdDIOjaouv3ag==
+  dependencies:
+    css-select "^5.1.0"
+    cssom "^0.5.0"
+    html-escaper "^3.0.3"
+    htmlparser2 "^8.0.1"
+    uhyphen "^0.1.0"
+
 linkedom@^0.14.9:
   version "0.14.9"
   resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0"
@@ -18151,6 +18162,11 @@ luxon@^2.3.1:
   resolved "https://registry.yarnpkg.com/luxon/-/luxon-2.3.1.tgz#f276b1b53fd9a740a60e666a541a7f6dbed4155a"
   integrity sha512-I8vnjOmhXsMSlNMZlMkSOvgrxKJl0uOsEzdGgGNZuZPaS9KlefpE9KV95QFftlJSC+1UyCC9/I69R02cz/zcCA==
 
+luxon@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929"
+  integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw==
+
 lz-string@^1.4.4:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"
@@ -24493,6 +24509,11 @@ underscore@^1.13.4, underscore@^1.9.1:
   resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee"
   integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ==
 
+underscore@^1.13.6:
+  version "1.13.6"
+  resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.6.tgz#04786a1f589dc6c09f761fc5f45b89e935136441"
+  integrity sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==
+
 undici@^4.9.3:
   version "4.14.1"
   resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9"