diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index ddd37e45c..9be264e5c 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -3,6 +3,57 @@ import axios from 'axios' import { DateTime } from 'luxon' import _ from 'underscore' +interface TweetIncludes { + users: { + id: string + name: string + profile_image_url: string + username: string + }[] + media?: { + preview_image_url: string + type: string + url: string + media_key: string + }[] +} + +interface TweetMeta { + result_count: number +} + +interface TweetData { + author_id: string + text: string + entities: { + urls: { + url: string + expanded_url: string + display_url: string + }[] + } + created_at: string + referenced_tweets: { + type: string + id: string + }[] + conversation_id: number + attachments?: { + media_keys: string[] + } +} + +interface Tweet { + data: TweetData + includes: TweetIncludes +} + +interface TweetThread { + data: TweetData[] + includes: TweetIncludes + meta: TweetMeta +} + const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ @@ -21,7 +72,36 @@ const getTweetFields = () => { return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}` } -const getTweetById = async (id: string) => { +// unroll recent tweet thread +const getTweetThread = async ( + id: string, + username: string +): Promise => { + const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/search/recent' + const apiUrl = new URL( + BASE_ENDPOINT + + '?query=' + + encodeURIComponent( + `conversation_id:${id} from:${username} to:${username}` + ) + + getTweetFields() + + '&max_results=100' + ) + + if (!TWITTER_BEARER_TOKEN) { + throw new Error('No Twitter bearer token found') + } + + const response = await axios.get(apiUrl.toString(), { + headers: { + Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, + redirect: 'follow', + }, + }) + return response.data +} + +const getTweetById = async (id: string): Promise => { const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/' const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields()) @@ -29,12 +109,14 @@ const getTweetById = async (id: string) => { throw new Error('No Twitter bearer token found') } - return axios.get(apiUrl.toString(), { + const response = await axios.get(apiUrl.toString(), { headers: { Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, redirect: 'follow', }, }) + + return response.data } const titleForAuthor = (author: { name: string }) => { @@ -63,103 +145,92 @@ export class TwitterHandler extends ContentHandler { } async preHandle(url: string, document?: Document): Promise { - console.log('prehandling twitter url', url) - const tweetId = tweetIdFromStatusUrl(url) if (!tweetId) { throw new Error('could not find tweet id in url') } - const tweetData = (await getTweetById(tweetId)).data as { - data: { - author_id: string - text: string - entities: { - urls: [ - { - url: string - expanded_url: string - display_url: string - } - ] - } - created_at: string - } - includes: { - users: [ - { - id: string - name: string - profile_image_url: string - username: string - } - ] - media: [ - { - preview_image_url: string - type: string - url: string - } - ] - } - } - const authorId = tweetData.data.author_id - const author = tweetData.includes.users.filter((u) => (u.id = authorId))[0] + const tweet = await getTweetById(tweetId) + const tweetData = tweet.data + const authorId = tweetData.author_id + const author = tweet.includes.users.filter((u) => (u.id = authorId))[0] // escape html entities in title const title = _.escape(titleForAuthor(author)) const authorImage = author.profile_image_url.replace('_normal', '_400x400') + const description = _.escape(tweetData.text) - let text = tweetData.data.text - if (tweetData.data.entities && tweetData.data.entities.urls) { - for (const urlObj of tweetData.data.entities.urls) { - text = text.replace( - urlObj.url, - `${urlObj.display_url}` + const tweets = [tweet] + // check if tweet is a thread + const thread = await getTweetThread(tweetId, author.username) + if (thread.meta.result_count > 0) { + // tweets are in reverse chronological order in the thread + for (const t of thread.data.reverse()) { + // get the tweet media if it exists + const media = thread.includes.media?.filter((m) => + t.attachments?.media_keys?.includes(m.media_key) ) + const tweet: Tweet = { + data: t, + includes: { + users: thread.includes.users, + media, + }, + } + tweets.push(tweet) } } - const front = ` -
-

${text}

- ` + let tweetsContent = '' + for (const tweet of tweets) { + const tweetData = tweet.data + let text = tweetData.text + if (tweetData.entities && tweetData.entities.urls) { + for (const urlObj of tweetData.entities.urls) { + text = text.replace( + urlObj.url, + `${urlObj.display_url}` + ) + } + } - let includesHtml = '' - if (tweetData.includes.media) { - includesHtml = tweetData.includes.media - .map((m) => { - const linkUrl = m.type == 'photo' ? m.url : url - const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url - const mediaOpen = ` + const includesHtml = + tweet.includes.media + ?.map((m) => { + const linkUrl = m.type == 'photo' ? m.url : url + const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url + return ` ` - return mediaOpen - }) - .join('\n') + }) + .join('\n') ?? '' + + tweetsContent += ` +

${text}

+ ${includesHtml} + ` } - const back = ` + const tweetUrl = ` — ${ author.username } ${author.name} ${formatTimestamp( - tweetData.data.created_at + tweetData.created_at )} -
` + const content = ` - + - ${front} - ${includesHtml} - ${back} +
+ ${tweetsContent} + ${tweetUrl} +
` return { content, url, title }