From 5bde463c94373f19a6b0f9815c5a1e0ee25c822a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 6 Oct 2022 16:56:12 +0800
Subject: [PATCH] Save unrolled recent twitter thread

---
 .../src/websites/twitter-handler.ts           | 170 +++++++++++-------
 1 file changed, 105 insertions(+), 65 deletions(-)

diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
index ddd37e45c..214c30409 100644
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@@ -3,6 +3,48 @@ import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
 
+interface TweetData {
+  author_id: string
+  text: string
+  entities: {
+    urls: [
+      {
+        url: string
+        expanded_url: string
+        display_url: string
+      }
+    ]
+  }
+  created_at: string
+  referenced_tweets: [
+    {
+      type: string
+      id: string
+    }
+  ]
+  includes: {
+    users: [
+      {
+        id: string
+        name: string
+        profile_image_url: string
+        username: string
+      }
+    ]
+    media: [
+      {
+        preview_image_url: string
+        type: string
+        url: string
+      }
+    ]
+  }
+}
+
+interface TweetThread {
+  data: TweetData[]
+}
+
 const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
 const TWITTER_URL_MATCH =
   /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
@@ -21,6 +63,30 @@ const getTweetFields = () => {
   return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`
 }
 
+// unroll recent tweet thread
+const getTweetThread = async (id: string): Promise<TweetThread> => {
+  const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent'
+  const apiUrl = new URL(
+    BASE_ENDPOINT +
+      '?query=' +
+      encodeURIComponent(`conversation_id:${id}`) +
+      '&' +
+      getTweetFields()
+  )
+
+  if (!TWITTER_BEARER_TOKEN) {
+    throw new Error('No Twitter bearer token found')
+  }
+
+  const response = await axios.get<TweetThread>(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: 'follow',
+    },
+  })
+  return response.data
+}
+
 const getTweetById = async (id: string) => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
   const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
@@ -69,97 +135,71 @@ export class TwitterHandler extends ContentHandler {
     if (!tweetId) {
       throw new Error('could not find tweet id in url')
     }
-    const tweetData = (await getTweetById(tweetId)).data as {
-      data: {
-        author_id: string
-        text: string
-        entities: {
-          urls: [
-            {
-              url: string
-              expanded_url: string
-              display_url: string
-            }
-          ]
-        }
-        created_at: string
-      }
-      includes: {
-        users: [
-          {
-            id: string
-            name: string
-            profile_image_url: string
-            username: string
-          }
-        ]
-        media: [
-          {
-            preview_image_url: string
-            type: string
-            url: string
-          }
-        ]
-      }
-    }
-    const authorId = tweetData.data.author_id
+    const tweetThread = await getTweetThread(tweetId)
+    const tweetData = tweetThread.data[0]
+    const authorId = tweetData.author_id
     const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0]
     // escape html entities in title
     const title = _.escape(titleForAuthor(author))
     const authorImage = author.profile_image_url.replace('_normal', '_400x400')
+    const description = _.escape(tweetData.text)
 
-    let text = tweetData.data.text
-    if (tweetData.data.entities && tweetData.data.entities.urls) {
-      for (const urlObj of tweetData.data.entities.urls) {
-        text = text.replace(
-          urlObj.url,
-          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
-        )
+    let tweetHtml = ''
+    for (const tweetData of tweetThread.data) {
+      let text = tweetData.text
+      if (tweetData.entities && tweetData.entities.urls) {
+        for (const urlObj of tweetData.entities.urls) {
+          text = text.replace(
+            urlObj.url,
+            `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
+          )
+        }
       }
-    }
 
-    const front = `
+      const front = `
     <div>
       <p>${text}</p>
     `
 
-    let includesHtml = ''
-    if (tweetData.includes.media) {
-      includesHtml = tweetData.includes.media
-        .map((m) => {
-          const linkUrl = m.type == 'photo' ? m.url : url
-          const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
-          const mediaOpen = `<a class="media-link" href=${linkUrl}>
+      let includesHtml = ''
+      if (tweetData.includes.media) {
+        includesHtml = tweetData.includes.media
+          .map((m) => {
+            const linkUrl = m.type == 'photo' ? m.url : url
+            const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
+            return `<a class="media-link" href=${linkUrl}>
           <picture>
             <img class="tweet-img" src=${previewUrl} />
           </picture>
           </a>`
-          return mediaOpen
-        })
-        .join('\n')
-    }
+          })
+          .join('\n')
+      }
 
-    const back = `
+      const back = `
        — <a href="https://twitter.com/${author.username}">${
-      author.username
-    }</a> ${author.name} <a href="${url}">${formatTimestamp(
-      tweetData.data.created_at
-    )}</a>
+        author.username
+      }</a> ${author.name} <a href="${url}">${formatTimestamp(
+        tweetData.created_at
+      )}</a>
     </div>
     `
+      tweetHtml += `
+        ${front}
+        ${includesHtml}
+        ${back}
+      `
+    }
+
     const content = `
     <head>
       <meta property="og:image" content="${authorImage}" />
       <meta property="og:image:secure_url" content="${authorImage}" />
       <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="${_.escape(
-        tweetData.data.text
-      )}" />
+      <meta property="og:description" content="${description}" />
     </head>
     <body>
-      ${front}
-      ${includesHtml}
-      ${back}
+      ${tweetHtml}
     </body>`
 
     return { content, url, title }