diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index 38d5608af..3dbe4c9d6 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -28,13 +28,13 @@ import { DerstandardHandler } from './websites/derstandard-handler' import { GitHubHandler } from './websites/github-handler' import { ImageHandler } from './websites/image-handler' import { MediumHandler } from './websites/medium-handler' -import { NitterHandler } from './websites/nitter-handler' import { PdfHandler } from './websites/pdf-handler' import { PipedVideoHandler } from './websites/piped-video-handler' import { ScrapingBeeHandler } from './websites/scrapingBee-handler' import { StackOverflowHandler } from './websites/stack-overflow-handler' import { TDotCoHandler } from './websites/t-dot-co-handler' import { TheAtlanticHandler } from './websites/the-atlantic-handler' +import { TwitterHandler } from './websites/twitter-handler' import { WeixinQqHandler } from './websites/weixin-qq-handler' import { WikipediaHandler } from './websites/wikipedia-handler' import { YoutubeHandler } from './websites/youtube-handler' @@ -81,8 +81,8 @@ const contentHandlers: ContentHandler[] = [ new EnergyWorldHandler(), new PipedVideoHandler(), new WeixinQqHandler(), - new NitterHandler(), new ZhihuHandler(), + new TwitterHandler(), ] const newsletterHandlers: ContentHandler[] = [ diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index e8457c3d7..bc26743eb 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -1,8 +1,8 @@ import axios from 'axios' +import { parseHTML } from 'linkedom' import { truncate } from 'lodash' import { DateTime } from 'luxon' import { Browser, BrowserContext } from 'puppeteer-core' -import _ from 'underscore' import { ContentHandler, PreHandleResult } from '../content-handler' interface TweetIncludes { @@ -56,6 +56,20 @@ interface Tweets { meta: TweetMeta } +interface EmbedTweet { + author_name: string + author_url: string + cache_age: string + height: number + html: string + provider_name: string + provider_url: string + type: string + url: string + version: string + width: number +} + const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ @@ -99,6 +113,15 @@ const getTweetThread = async (conversationId: string): Promise => { return response.data } +const getEmbedTweet = async (url: string): Promise => { + const BASE_ENDPOINT = 'https://publish.twitter.com/oembed' + const embedUrl = new URL(BASE_ENDPOINT + '?url=' + encodeURIComponent(url)) + + const response = await axios.get(embedUrl.toString()) + + return response.data +} + const getTweetById = async (id: string): Promise => { const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/' const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields()) @@ -135,8 +158,8 @@ const getTweetsByIds = async (ids: string[]): Promise => { return response.data } -const titleForTweet = (author: { name: string }, text: string) => { - return `${author.name} on Twitter: ${truncate(text.replace(/http\S+/, ''), { +const titleForTweet = (author: string, text: string) => { + return `${author} on Twitter: ${truncate(text.replace(/http\S+/, ''), { length: 100, })}` } @@ -297,92 +320,41 @@ export class TwitterHandler extends ContentHandler { } shouldPreHandle(url: string): boolean { - return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) + return TWITTER_URL_MATCH.test(url.toString()) } - async preHandle(url: string, browser: Browser): Promise { - const tweetId = tweetIdFromStatusUrl(url) - if (!tweetId) { - throw new Error('could not find tweet id in url') - } - let tweet = await getTweetById(tweetId) - const conversationId = tweet.data.conversation_id - if (conversationId !== tweetId) { - // this is a reply, so we need to get the referenced tweet - tweet = await getTweetById(conversationId) - } + async preHandle(url: string): Promise { + const embedTweet = await getEmbedTweet(url) + console.log('embedTweet', embedTweet) + const html = embedTweet.html - const tweetData = tweet.data - const authorId = tweetData.author_id - const author = tweet.includes.users.filter((u) => (u.id = authorId))[0] - // escape html entities in title - const title = titleForTweet(author, tweetData.text) - const escapedTitle = _.escape(title) - const authorImage = author.profile_image_url.replace('_normal', '_400x400') - const description = _.escape(tweetData.text) + const dom = parseHTML(html).document - // use puppeteer to get all tweet replies in the thread - const tweets = await getOldTweets(browser, conversationId, author.username) - - let tweetsContent = '' - for (const tweet of tweets) { - const tweetData = tweet.data - let text = tweetData.text - if (tweetData.entities && tweetData.entities.urls) { - for (const urlObj of tweetData.entities.urls) { - text = text.replace( - urlObj.url, - `${urlObj.display_url}` - ) - } - } - - const includesHtml = - tweet.includes.media - ?.map((m) => { - const linkUrl = m.type == 'photo' ? m.url : url - const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url - return ` - - - - ` - }) - .join('\n') ?? '' - - tweetsContent += ` -

${text}

- ${includesHtml} - ` - } - - const tweetUrl = ` - — ${ - author.username - } ${formatTimestamp(tweetData.created_at)} - ` + const tweetText = dom.querySelector('p')?.textContent ?? '' + const title = titleForTweet(embedTweet.author_name, tweetText) + const publisedDate = + dom.querySelector('a[href*="/status/"]')?.textContent ?? '' const content = ` - - - - - - - - - - - -
- ${tweetsContent} - ${tweetUrl} -
- -` + + + + + + + + + +
+ ${embedTweet.html} +
+ + ` - return { content, url, title } + return { + content, + url, + title, + } } }