Merge pull request #1286 from omnivore-app/feature/save-twitter-threads

Save unrolled recent twitter thread

packages/content-handler/src/websites/twitter-handler.ts

@@ -3,6 +3,57 @@ import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
 
+interface TweetIncludes {
+  users: {
+    id: string
+    name: string
+    profile_image_url: string
+    username: string
+  }[]
+  media?: {
+    preview_image_url: string
+    type: string
+    url: string
+    media_key: string
+  }[]
+}
+
+interface TweetMeta {
+  result_count: number
+}
+
+interface TweetData {
+  author_id: string
+  text: string
+  entities: {
+    urls: {
+      url: string
+      expanded_url: string
+      display_url: string
+    }[]
+  }
+  created_at: string
+  referenced_tweets: {
+    type: string
+    id: string
+  }[]
+  conversation_id: number
+  attachments?: {
+    media_keys: string[]
+  }
+}
+
+interface Tweet {
+  data: TweetData
+  includes: TweetIncludes
+}
+
+interface TweetThread {
+  data: TweetData[]
+  includes: TweetIncludes
+  meta: TweetMeta
+}
+
 const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
 const TWITTER_URL_MATCH =
   /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
@@ -21,7 +72,36 @@ const getTweetFields = () => {
   return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`
 }
 
-const getTweetById = async (id: string) => {
+// unroll recent tweet thread
+const getTweetThread = async (
+  id: string,
+  username: string
+): Promise<TweetThread> => {
+  const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent'
+  const apiUrl = new URL(
+    BASE_ENDPOINT +
+      '?query=' +
+      encodeURIComponent(
+        `conversation_id:${id} from:${username} to:${username}`
+      ) +
+      getTweetFields() +
+      '&max_results=100'
+  )
+
+  if (!TWITTER_BEARER_TOKEN) {
+    throw new Error('No Twitter bearer token found')
+  }
+
+  const response = await axios.get<TweetThread>(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: 'follow',
+    },
+  })
+  return response.data
+}
+
+const getTweetById = async (id: string): Promise<Tweet> => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
   const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
 
@@ -29,12 +109,14 @@ const getTweetById = async (id: string) => {
     throw new Error('No Twitter bearer token found')
   }
 
-  return axios.get(apiUrl.toString(), {
+  const response = await axios.get<Tweet>(apiUrl.toString(), {
     headers: {
       Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
       redirect: 'follow',
     },
   })
+
+  return response.data
 }
 
 const titleForAuthor = (author: { name: string }) => {
@@ -63,103 +145,92 @@ export class TwitterHandler extends ContentHandler {
   }
 
   async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
-    console.log('prehandling twitter url', url)
-
     const tweetId = tweetIdFromStatusUrl(url)
     if (!tweetId) {
       throw new Error('could not find tweet id in url')
     }
-    const tweetData = (await getTweetById(tweetId)).data as {
-      data: {
-        author_id: string
-        text: string
-        entities: {
-          urls: [
-            {
-              url: string
-              expanded_url: string
-              display_url: string
-            }
-          ]
-        }
-        created_at: string
-      }
-      includes: {
-        users: [
-          {
-            id: string
-            name: string
-            profile_image_url: string
-            username: string
-          }
-        ]
-        media: [
-          {
-            preview_image_url: string
-            type: string
-            url: string
-          }
-        ]
-      }
-    }
-    const authorId = tweetData.data.author_id
-    const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0]
+    const tweet = await getTweetById(tweetId)
+    const tweetData = tweet.data
+    const authorId = tweetData.author_id
+    const author = tweet.includes.users.filter((u) => (u.id = authorId))[0]
     // escape html entities in title
     const title = _.escape(titleForAuthor(author))
     const authorImage = author.profile_image_url.replace('_normal', '_400x400')
+    const description = _.escape(tweetData.text)
 
-    let text = tweetData.data.text
-    if (tweetData.data.entities && tweetData.data.entities.urls) {
-      for (const urlObj of tweetData.data.entities.urls) {
-        text = text.replace(
-          urlObj.url,
-          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
+    const tweets = [tweet]
+    // check if tweet is a thread
+    const thread = await getTweetThread(tweetId, author.username)
+    if (thread.meta.result_count > 0) {
+      // tweets are in reverse chronological order in the thread
+      for (const t of thread.data.reverse()) {
+        // get the tweet media if it exists
+        const media = thread.includes.media?.filter((m) =>
+          t.attachments?.media_keys?.includes(m.media_key)
         )
+        const tweet: Tweet = {
+          data: t,
+          includes: {
+            users: thread.includes.users,
+            media,
+          },
+        }
+        tweets.push(tweet)
       }
     }
 
-    const front = `
-    <div>
-      <p>${text}</p>
-    `
+    let tweetsContent = ''
+    for (const tweet of tweets) {
+      const tweetData = tweet.data
+      let text = tweetData.text
+      if (tweetData.entities && tweetData.entities.urls) {
+        for (const urlObj of tweetData.entities.urls) {
+          text = text.replace(
+            urlObj.url,
+            `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
+          )
+        }
+      }
 
-    let includesHtml = ''
-    if (tweetData.includes.media) {
-      includesHtml = tweetData.includes.media
-        .map((m) => {
-          const linkUrl = m.type == 'photo' ? m.url : url
-          const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
-          const mediaOpen = `<a class="media-link" href=${linkUrl}>
+      const includesHtml =
+        tweet.includes.media
+          ?.map((m) => {
+            const linkUrl = m.type == 'photo' ? m.url : url
+            const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
+            return `<a class="media-link" href=${linkUrl}>
           <picture>
             <img class="tweet-img" src=${previewUrl} />
           </picture>
           </a>`
-          return mediaOpen
-        })
-        .join('\n')
+          })
+          .join('\n') ?? ''
+
+      tweetsContent += `
+      <p>${text}</p>
+      ${includesHtml}
+    `
     }
 
-    const back = `
+    const tweetUrl = `
        — <a href="https://twitter.com/${author.username}">${
       author.username
     }</a> ${author.name} <a href="${url}">${formatTimestamp(
-      tweetData.data.created_at
+      tweetData.created_at
     )}</a>
-    </div>
     `
+
     const content = `
     <head>
       <meta property="og:image" content="${authorImage}" />
       <meta property="og:image:secure_url" content="${authorImage}" />
       <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="${_.escape(
-        tweetData.data.text
-      )}" />
+      <meta property="og:description" content="${description}" />
     </head>
     <body>
-      ${front}
-      ${includesHtml}
-      ${back}
+      <div>
+        ${tweetsContent}
+        ${tweetUrl}
+      </div>
     </body>`
 
     return { content, url, title }