From 5bde463c94373f19a6b0f9815c5a1e0ee25c822a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 6 Oct 2022 16:56:12 +0800
Subject: [PATCH 1/3] Save unrolled recent twitter thread

---
 .../src/websites/twitter-handler.ts           | 170 +++++++++++-------
 1 file changed, 105 insertions(+), 65 deletions(-)

diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
index ddd37e45c..214c30409 100644
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@@ -3,6 +3,48 @@ import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
 
+interface TweetData {
+  author_id: string
+  text: string
+  entities: {
+    urls: [
+      {
+        url: string
+        expanded_url: string
+        display_url: string
+      }
+    ]
+  }
+  created_at: string
+  referenced_tweets: [
+    {
+      type: string
+      id: string
+    }
+  ]
+  includes: {
+    users: [
+      {
+        id: string
+        name: string
+        profile_image_url: string
+        username: string
+      }
+    ]
+    media: [
+      {
+        preview_image_url: string
+        type: string
+        url: string
+      }
+    ]
+  }
+}
+
+interface TweetThread {
+  data: TweetData[]
+}
+
 const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
 const TWITTER_URL_MATCH =
   /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
@@ -21,6 +63,30 @@ const getTweetFields = () => {
   return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`
 }
 
+// unroll recent tweet thread
+const getTweetThread = async (id: string): Promise<TweetThread> => {
+  const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent'
+  const apiUrl = new URL(
+    BASE_ENDPOINT +
+      '?query=' +
+      encodeURIComponent(`conversation_id:${id}`) +
+      '&' +
+      getTweetFields()
+  )
+
+  if (!TWITTER_BEARER_TOKEN) {
+    throw new Error('No Twitter bearer token found')
+  }
+
+  const response = await axios.get<TweetThread>(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: 'follow',
+    },
+  })
+  return response.data
+}
+
 const getTweetById = async (id: string) => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
   const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
@@ -69,97 +135,71 @@ export class TwitterHandler extends ContentHandler {
     if (!tweetId) {
       throw new Error('could not find tweet id in url')
     }
-    const tweetData = (await getTweetById(tweetId)).data as {
-      data: {
-        author_id: string
-        text: string
-        entities: {
-          urls: [
-            {
-              url: string
-              expanded_url: string
-              display_url: string
-            }
-          ]
-        }
-        created_at: string
-      }
-      includes: {
-        users: [
-          {
-            id: string
-            name: string
-            profile_image_url: string
-            username: string
-          }
-        ]
-        media: [
-          {
-            preview_image_url: string
-            type: string
-            url: string
-          }
-        ]
-      }
-    }
-    const authorId = tweetData.data.author_id
+    const tweetThread = await getTweetThread(tweetId)
+    const tweetData = tweetThread.data[0]
+    const authorId = tweetData.author_id
     const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0]
     // escape html entities in title
     const title = _.escape(titleForAuthor(author))
     const authorImage = author.profile_image_url.replace('_normal', '_400x400')
+    const description = _.escape(tweetData.text)
 
-    let text = tweetData.data.text
-    if (tweetData.data.entities && tweetData.data.entities.urls) {
-      for (const urlObj of tweetData.data.entities.urls) {
-        text = text.replace(
-          urlObj.url,
-          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
-        )
+    let tweetHtml = ''
+    for (const tweetData of tweetThread.data) {
+      let text = tweetData.text
+      if (tweetData.entities && tweetData.entities.urls) {
+        for (const urlObj of tweetData.entities.urls) {
+          text = text.replace(
+            urlObj.url,
+            `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
+          )
+        }
       }
-    }
 
-    const front = `
+      const front = `
     <div>
       <p>${text}</p>
     `
 
-    let includesHtml = ''
-    if (tweetData.includes.media) {
-      includesHtml = tweetData.includes.media
-        .map((m) => {
-          const linkUrl = m.type == 'photo' ? m.url : url
-          const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
-          const mediaOpen = `<a class="media-link" href=${linkUrl}>
+      let includesHtml = ''
+      if (tweetData.includes.media) {
+        includesHtml = tweetData.includes.media
+          .map((m) => {
+            const linkUrl = m.type == 'photo' ? m.url : url
+            const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
+            return `<a class="media-link" href=${linkUrl}>
           <picture>
             <img class="tweet-img" src=${previewUrl} />
           </picture>
           </a>`
-          return mediaOpen
-        })
-        .join('\n')
-    }
+          })
+          .join('\n')
+      }
 
-    const back = `
+      const back = `
        — <a href="https://twitter.com/${author.username}">${
-      author.username
-    }</a> ${author.name} <a href="${url}">${formatTimestamp(
-      tweetData.data.created_at
-    )}</a>
+        author.username
+      }</a> ${author.name} <a href="${url}">${formatTimestamp(
+        tweetData.created_at
+      )}</a>
     </div>
     `
+      tweetHtml += `
+        ${front}
+        ${includesHtml}
+        ${back}
+      `
+    }
+
     const content = `
     <head>
       <meta property="og:image" content="${authorImage}" />
       <meta property="og:image:secure_url" content="${authorImage}" />
       <meta property="og:title" content="${title}" />
-      <meta property="og:description" content="${_.escape(
-        tweetData.data.text
-      )}" />
+      <meta property="og:description" content="${description}" />
     </head>
     <body>
-      ${front}
-      ${includesHtml}
-      ${back}
+      ${tweetHtml}
     </body>`
 
     return { content, url, title }

From a8f964f6157de11228abb232addcb97fe6f652e8 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 7 Oct 2022 13:47:20 +0800
Subject: [PATCH 2/3] Save only tweets in the thread by the original author

---
 .../src/websites/twitter-handler.ts           | 130 ++++++++++--------
 1 file changed, 75 insertions(+), 55 deletions(-)

diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
index 214c30409..a1d9c63e1 100644
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@@ -3,6 +3,28 @@ import axios from 'axios'
 import { DateTime } from 'luxon'
 import _ from 'underscore'
 
+interface TweetIncludes {
+  users: [
+    {
+      id: string
+      name: string
+      profile_image_url: string
+      username: string
+    }
+  ]
+  media?: [
+    {
+      preview_image_url: string
+      type: string
+      url: string
+    }
+  ]
+}
+
+interface TweetMeta {
+  result_count: number
+}
+
 interface TweetData {
   author_id: string
   text: string
@@ -22,27 +44,18 @@ interface TweetData {
       id: string
     }
   ]
-  includes: {
-    users: [
-      {
-        id: string
-        name: string
-        profile_image_url: string
-        username: string
-      }
-    ]
-    media: [
-      {
-        preview_image_url: string
-        type: string
-        url: string
-      }
-    ]
-  }
+  conversation_id: number
+}
+
+interface Tweet {
+  data: TweetData
+  includes: TweetIncludes
 }
 
 interface TweetThread {
   data: TweetData[]
+  includes: TweetIncludes
+  meta: TweetMeta
 }
 
 const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN
@@ -64,14 +77,19 @@ const getTweetFields = () => {
 }
 
 // unroll recent tweet thread
-const getTweetThread = async (id: string): Promise<TweetThread> => {
+const getTweetThread = async (
+  id: string,
+  username: string
+): Promise<TweetThread> => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent'
   const apiUrl = new URL(
     BASE_ENDPOINT +
       '?query=' +
-      encodeURIComponent(`conversation_id:${id}`) +
-      '&' +
-      getTweetFields()
+      encodeURIComponent(
+        `conversation_id:${id} from:${username} to:${username}`
+      ) +
+      getTweetFields() +
+      '&max_results=100'
   )
 
   if (!TWITTER_BEARER_TOKEN) {
@@ -87,7 +105,7 @@ const getTweetThread = async (id: string): Promise<TweetThread> => {
   return response.data
 }
 
-const getTweetById = async (id: string) => {
+const getTweetById = async (id: string): Promise<Tweet> => {
   const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/'
   const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
 
@@ -95,12 +113,14 @@ const getTweetById = async (id: string) => {
     throw new Error('No Twitter bearer token found')
   }
 
-  return axios.get(apiUrl.toString(), {
+  const response = await axios.get<Tweet>(apiUrl.toString(), {
     headers: {
       Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
       redirect: 'follow',
     },
   })
+
+  return response.data
 }
 
 const titleForAuthor = (author: { name: string }) => {
@@ -129,23 +149,28 @@ export class TwitterHandler extends ContentHandler {
   }
 
   async preHandle(url: string, document?: Document): Promise<PreHandleResult> {
-    console.log('prehandling twitter url', url)
-
     const tweetId = tweetIdFromStatusUrl(url)
     if (!tweetId) {
       throw new Error('could not find tweet id in url')
     }
-    const tweetThread = await getTweetThread(tweetId)
-    const tweetData = tweetThread.data[0]
+    const tweet = await getTweetById(tweetId)
+    const tweetData = tweet.data
     const authorId = tweetData.author_id
-    const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0]
+    const author = tweet.includes.users.filter((u) => (u.id = authorId))[0]
     // escape html entities in title
     const title = _.escape(titleForAuthor(author))
     const authorImage = author.profile_image_url.replace('_normal', '_400x400')
     const description = _.escape(tweetData.text)
 
-    let tweetHtml = ''
-    for (const tweetData of tweetThread.data) {
+    const tweetsData = [tweet.data]
+    // check if tweet is a thread
+    const thread = await getTweetThread(tweetId, author.username)
+    if (thread.meta.result_count > 0) {
+      tweetsData.push(...thread.data)
+    }
+
+    let front = ''
+    for (const tweetData of tweetsData) {
       let text = tweetData.text
       if (tweetData.entities && tweetData.entities.urls) {
         for (const urlObj of tweetData.entities.urls) {
@@ -156,40 +181,31 @@ export class TwitterHandler extends ContentHandler {
         }
       }
 
-      const front = `
-    <div>
+      front += `
       <p>${text}</p>
     `
+    }
 
-      let includesHtml = ''
-      if (tweetData.includes.media) {
-        includesHtml = tweetData.includes.media
-          .map((m) => {
-            const linkUrl = m.type == 'photo' ? m.url : url
-            const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
-            return `<a class="media-link" href=${linkUrl}>
+    const includesHtml =
+      tweet.includes.media
+        ?.map((m) => {
+          const linkUrl = m.type == 'photo' ? m.url : url
+          const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
+          return `<a class="media-link" href=${linkUrl}>
           <picture>
             <img class="tweet-img" src=${previewUrl} />
           </picture>
           </a>`
-          })
-          .join('\n')
-      }
+        })
+        .join('\n') ?? ''
 
-      const back = `
+    const back = `
        — <a href="https://twitter.com/${author.username}">${
-        author.username
-      }</a> ${author.name} <a href="${url}">${formatTimestamp(
-        tweetData.created_at
-      )}</a>
-    </div>
+      author.username
+    }</a> ${author.name} <a href="${url}">${formatTimestamp(
+      tweetData.created_at
+    )}</a>
     `
-      tweetHtml += `
-        ${front}
-        ${includesHtml}
-        ${back}
-      `
-    }
 
     const content = `
     <head>
@@ -199,7 +215,11 @@ export class TwitterHandler extends ContentHandler {
       <meta property="og:description" content="${description}" />
     </head>
     <body>
-      ${tweetHtml}
+      <div>
+        ${front}
+        ${includesHtml}
+        ${back}
+      </div>
     </body>`
 
     return { content, url, title }

From 810df2ad8cbe2a7b695fa17a94d0b8a8be4b6c3a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 7 Oct 2022 16:31:56 +0800
Subject: [PATCH 3/3] Fix the tweets in reverse order and media file not
 showing

---
 .../src/websites/twitter-handler.ts           | 109 ++++++++++--------
 1 file changed, 60 insertions(+), 49 deletions(-)

diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
index a1d9c63e1..9be264e5c 100644
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@@ -4,21 +4,18 @@ import { DateTime } from 'luxon'
 import _ from 'underscore'
 
 interface TweetIncludes {
-  users: [
-    {
-      id: string
-      name: string
-      profile_image_url: string
-      username: string
-    }
-  ]
-  media?: [
-    {
-      preview_image_url: string
-      type: string
-      url: string
-    }
-  ]
+  users: {
+    id: string
+    name: string
+    profile_image_url: string
+    username: string
+  }[]
+  media?: {
+    preview_image_url: string
+    type: string
+    url: string
+    media_key: string
+  }[]
 }
 
 interface TweetMeta {
@@ -29,22 +26,21 @@ interface TweetData {
   author_id: string
   text: string
   entities: {
-    urls: [
-      {
-        url: string
-        expanded_url: string
-        display_url: string
-      }
-    ]
+    urls: {
+      url: string
+      expanded_url: string
+      display_url: string
+    }[]
   }
   created_at: string
-  referenced_tweets: [
-    {
-      type: string
-      id: string
-    }
-  ]
+  referenced_tweets: {
+    type: string
+    id: string
+  }[]
   conversation_id: number
+  attachments?: {
+    media_keys: string[]
+  }
 }
 
 interface Tweet {
@@ -162,15 +158,30 @@ export class TwitterHandler extends ContentHandler {
     const authorImage = author.profile_image_url.replace('_normal', '_400x400')
     const description = _.escape(tweetData.text)
 
-    const tweetsData = [tweet.data]
+    const tweets = [tweet]
     // check if tweet is a thread
     const thread = await getTweetThread(tweetId, author.username)
     if (thread.meta.result_count > 0) {
-      tweetsData.push(...thread.data)
+      // tweets are in reverse chronological order in the thread
+      for (const t of thread.data.reverse()) {
+        // get the tweet media if it exists
+        const media = thread.includes.media?.filter((m) =>
+          t.attachments?.media_keys?.includes(m.media_key)
+        )
+        const tweet: Tweet = {
+          data: t,
+          includes: {
+            users: thread.includes.users,
+            media,
+          },
+        }
+        tweets.push(tweet)
+      }
     }
 
-    let front = ''
-    for (const tweetData of tweetsData) {
+    let tweetsContent = ''
+    for (const tweet of tweets) {
+      const tweetData = tweet.data
       let text = tweetData.text
       if (tweetData.entities && tweetData.entities.urls) {
         for (const urlObj of tweetData.entities.urls) {
@@ -181,25 +192,26 @@ export class TwitterHandler extends ContentHandler {
         }
       }
 
-      front += `
-      <p>${text}</p>
-    `
-    }
-
-    const includesHtml =
-      tweet.includes.media
-        ?.map((m) => {
-          const linkUrl = m.type == 'photo' ? m.url : url
-          const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
-          return `<a class="media-link" href=${linkUrl}>
+      const includesHtml =
+        tweet.includes.media
+          ?.map((m) => {
+            const linkUrl = m.type == 'photo' ? m.url : url
+            const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url
+            return `<a class="media-link" href=${linkUrl}>
           <picture>
             <img class="tweet-img" src=${previewUrl} />
           </picture>
           </a>`
-        })
-        .join('\n') ?? ''
+          })
+          .join('\n') ?? ''
 
-    const back = `
+      tweetsContent += `
+      <p>${text}</p>
+      ${includesHtml}
+    `
+    }
+
+    const tweetUrl = `
        — <a href="https://twitter.com/${author.username}">${
       author.username
     }</a> ${author.name} <a href="${url}">${formatTimestamp(
@@ -216,9 +228,8 @@ export class TwitterHandler extends ContentHandler {
     </head>
     <body>
       <div>
-        ${front}
-        ${includesHtml}
-        ${back}
+        ${tweetsContent}
+        ${tweetUrl}
       </div>
     </body>`