Merge pull request #3914 from omnivore-app/feature/embed-tweet

feat: embed tweets if user saves tweets

packages/content-handler/src/index.ts

@@ -28,13 +28,13 @@ import { DerstandardHandler } from './websites/derstandard-handler'
 import { GitHubHandler } from './websites/github-handler'
 import { ImageHandler } from './websites/image-handler'
 import { MediumHandler } from './websites/medium-handler'
-import { NitterHandler } from './websites/nitter-handler'
 import { PdfHandler } from './websites/pdf-handler'
 import { PipedVideoHandler } from './websites/piped-video-handler'
 import { ScrapingBeeHandler } from './websites/scrapingBee-handler'
 import { StackOverflowHandler } from './websites/stack-overflow-handler'
 import { TDotCoHandler } from './websites/t-dot-co-handler'
 import { TheAtlanticHandler } from './websites/the-atlantic-handler'
+import { TwitterHandler } from './websites/twitter-handler'
 import { WeixinQqHandler } from './websites/weixin-qq-handler'
 import { WikipediaHandler } from './websites/wikipedia-handler'
 import { YoutubeHandler } from './websites/youtube-handler'
@@ -81,8 +81,8 @@ const contentHandlers: ContentHandler[] = [
   new EnergyWorldHandler(),
   new PipedVideoHandler(),
   new WeixinQqHandler(),
-  new NitterHandler(),
   new ZhihuHandler(),
+  new TwitterHandler(),
 ]
 
 const newsletterHandlers: ContentHandler[] = [

packages/content-handler/src/websites/twitter-handler.ts

@@ -1,8 +1,8 @@
 import axios from 'axios'
+import { parseHTML } from 'linkedom'
 import { truncate } from 'lodash'
 import { DateTime } from 'luxon'
 import { Browser, BrowserContext } from 'puppeteer-core'
-import _ from 'underscore'
 import { ContentHandler, PreHandleResult } from '../content-handler'
 
 interface TweetIncludes {
@@ -56,6 +56,20 @@ interface Tweets {
   meta: TweetMeta
 }
 
+interface EmbedTweet {
+  author_name: string
+  author_url: string
+  cache_age: string
+  height: number
+  html: string
+  provider_name: string
+  provider_url: string
+  type: string
+  url: string
+  version: string
+  width: number
+}
+
 const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
 const TWITTER_URL_MATCH =
   /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
@@ -99,6 +113,15 @@ const getTweetThread = async (conversationId: string): Promise<Tweets> => {
   return response.data
 }
 
+const getEmbedTweet = async (url: string): Promise<EmbedTweet> => {
+  const BASE_ENDPOINT = 'https://publish.twitter.com/oembed'
+  const embedUrl = new URL(BASE_ENDPOINT + '?url=' + encodeURIComponent(url))
+
+  const response = await axios.get<EmbedTweet>(embedUrl.toString())
+
+  return response.data
+}
+
 const getTweetById = async (id: string): Promise<Tweet> => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
   const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
@@ -135,8 +158,8 @@ const getTweetsByIds = async (ids: string[]): Promise<Tweets> => {
   return response.data
 }
 
-const titleForTweet = (author: { name: string }, text: string) => {
-  return `${author.name} on Twitter: ${truncate(text.replace(/http\S+/, ''), {
+const titleForTweet = (author: string, text: string) => {
+  return `${author} on Twitter: ${truncate(text.replace(/http\S+/, ''), {
     length: 100,
   })}`
 }
@@ -297,92 +320,41 @@ export class TwitterHandler extends ContentHandler {
   }
 
   shouldPreHandle(url: string): boolean {
-    return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
+    return TWITTER_URL_MATCH.test(url.toString())
   }
 
-  async preHandle(url: string, browser: Browser): Promise<PreHandleResult> {
-    const tweetId = tweetIdFromStatusUrl(url)
-    if (!tweetId) {
-      throw new Error('could not find tweet id in url')
-    }
-    let tweet = await getTweetById(tweetId)
-    const conversationId = tweet.data.conversation_id
-    if (conversationId !== tweetId) {
-      // this is a reply, so we need to get the referenced tweet
-      tweet = await getTweetById(conversationId)
-    }
+  async preHandle(url: string): Promise<PreHandleResult> {
+    const embedTweet = await getEmbedTweet(url)
+    console.log('embedTweet', embedTweet)
+    const html = embedTweet.html
 
-    const tweetData = tweet.data
-    const authorId = tweetData.author_id
-    const author = tweet.includes.users.filter((u) => (u.id = authorId))[0]
-    // escape html entities in title
-    const title = titleForTweet(author, tweetData.text)
-    const escapedTitle = _.escape(title)
-    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
-    const description = _.escape(tweetData.text)
+    const dom = parseHTML(html).document
 
-    // use puppeteer to get all tweet replies in the thread
-    const tweets = await getOldTweets(browser, conversationId, author.username)
-
-    let tweetsContent = ''
-    for (const tweet of tweets) {
-      const tweetData = tweet.data
-      let text = tweetData.text
-      if (tweetData.entities && tweetData.entities.urls) {
-        for (const urlObj of tweetData.entities.urls) {
-          text = text.replace(
-            urlObj.url,
-            `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
-          )
-        }
-      }
-
-      const includesHtml =
-        tweet.includes.media
-          ?.map((m) => {
-            const linkUrl = m.type == 'photo' ? m.url : url
-            const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
-            return `<a class="media-link" href=${linkUrl}>
-          <picture>
-            <img class="tweet-img" src=${previewUrl} />
-          </picture>
-          </a>`
-          })
-          .join('\n') ?? ''
-
-      tweetsContent += `
-      <p>${text}</p>
-      ${includesHtml}
-    `
-    }
-
-    const tweetUrl = `
-       — <a href="https://twitter.com/${author.username}">${
-      author.username
-    }</a> <span itemscope itemtype="https://schema.org/Person" itemprop="author">${
-      author.name
-    }</span> <a href="${url}">${formatTimestamp(tweetData.created_at)}</a>
-    `
+    const tweetText = dom.querySelector('p')?.textContent ?? ''
+    const title = titleForTweet(embedTweet.author_name, tweetText)
+    const publisedDate =
+      dom.querySelector('a[href*="/status/"]')?.textContent ?? ''
 
     const content = `
-<html>
-    <head>
-      <meta property="og:image" content="${authorImage}" />
-      <meta property="og:image:secure_url" content="${authorImage}" />
-      <meta property="og:title" content="${escapedTitle}" />
-      <meta property="og:description" content="${description}" />
-      <meta property="article:published_time" content="${tweetData.created_at}" />
-      <meta property="og:site_name" content="Twitter" />
-      <meta property="og:type" content="tweet" />
-    </head>
-    <body>
-      <div>
-        ${tweetsContent}
-        ${tweetUrl}
-      </div>
-    </body>
-</html>`
+      <html>
+          <head>
+            <meta property="og:site_name" content="X (formerly Twitter)" />
+            <meta property="og:type" content="tweet" />
+            <meta property="dc:creator" content="${embedTweet.author_name}" />
+            <meta property="twitter:description" content="${tweetText}" />
+            <meta property="article:published_time" content="${publisedDate}" />
+          </head>
+          <body>
+            <div>
+              ${embedTweet.html}
+            </div>
+          </body>
+      </html>`
 
-    return { content, url, title }
+    return {
+      content,
+      url,
+      title,
+    }
   }
 }