diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index c8f5d7baf..57d113a0d 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -27,12 +27,12 @@ import { DerstandardHandler } from './websites/derstandard-handler' import { GitHubHandler } from './websites/github-handler' import { ImageHandler } from './websites/image-handler' import { MediumHandler } from './websites/medium-handler' +import { NitterHandler } from './websites/nitter-handler' import { PdfHandler } from './websites/pdf-handler' import { PipedVideoHandler } from './websites/piped-video-handler' import { ScrapingBeeHandler } from './websites/scrapingBee-handler' import { StackOverflowHandler } from './websites/stack-overflow-handler' import { TDotCoHandler } from './websites/t-dot-co-handler' -import { TwitterHandler } from './websites/twitter-handler' import { WeixinQqHandler } from './websites/weixin-qq-handler' import { WikipediaHandler } from './websites/wikipedia-handler' import { YoutubeHandler } from './websites/youtube-handler' @@ -64,7 +64,6 @@ const contentHandlers: ContentHandler[] = [ new PdfHandler(), new ScrapingBeeHandler(), new TDotCoHandler(), - new TwitterHandler(), new YoutubeHandler(), new WikipediaHandler(), new GitHubHandler(), @@ -77,6 +76,7 @@ const contentHandlers: ContentHandler[] = [ new EnergyWorldHandler(), new PipedVideoHandler(), new WeixinQqHandler(), + new NitterHandler(), ] const newsletterHandlers: ContentHandler[] = [ diff --git a/packages/content-handler/src/websites/nitter-handler.ts b/packages/content-handler/src/websites/nitter-handler.ts new file mode 100644 index 000000000..f5c5a8eab --- /dev/null +++ b/packages/content-handler/src/websites/nitter-handler.ts @@ -0,0 +1,305 @@ +import { parseHTML } from 'linkedom' +import _, { truncate } from 'lodash' +import { DateTime } from 'luxon' +import { Browser, BrowserContext } from 'puppeteer-core' +import { ContentHandler, PreHandleResult } from '../content-handler' + +interface Tweet { + url: string + author: { + username: string + name: string + profileImageUrl: string + } + text: string + entities: { + urls: { + url: string + display_url: string + }[] + photos: string[] + videos: string[] + } + createdAt: string +} + +export class NitterHandler extends ContentHandler { + // matches twitter.com and nitter.net urls + URL_MATCH = + /((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ + ADDRESS = 'https://nitter.net' + + constructor() { + super() + this.name = 'Nitter' + } + + async getTweets(browser: Browser, username: string, tweetId: string) { + const url = `${this.ADDRESS}/${username}/status/${tweetId}` + + async function genTweets(address: string): Promise { + function authorParser(header: Element) { + const avatar = header + .querySelector('.tweet-avatar img') + ?.getAttribute('src') + if (!avatar) { + return null + } + const name = header.querySelector('.fullname')?.getAttribute('title') + if (!name) { + return null + } + return { + avatar, + name, + } + } + + function dateParser(tweetDate: string) { + const validDateTime = tweetDate.replace(' · ', ' ') + + return new Date(validDateTime).toISOString() + } + + function attachmentParser(attachments: Element | null) { + if (!attachments) return { photos: [], videos: [] } + + const photos = Array.from(attachments.querySelectorAll('img')).map( + (i) => i.getAttribute('src') ?? '' + ) + const videos = Array.from(attachments.querySelectorAll('source')).map( + (i) => i.getAttribute('src') ?? '' + ) + return { + photos, + videos, + } + } + + function parseTweet(tweet: Element): Tweet | null { + const header = tweet.querySelector('.tweet-header') + if (!header) { + return null + } + const author = authorParser(header) + if (!author) { + return null + } + + const body = tweet.querySelector('.tweet-body') + if (!body) { + return null + } + + const tweetDate = body + .querySelector('.tweet-date a') + ?.getAttribute('title') + if (!tweetDate) { + return null + } + const createdAt = dateParser(tweetDate) + + const content = body.querySelector('.tweet-content') + if (!content) { + return null + } + const text = content.textContent ?? '' + const urls = Array.from(content.querySelectorAll('a')).map((a) => ({ + url: a.getAttribute('href') ?? '', + display_url: a.textContent ?? '', + })) + + const attachments = body.querySelector('.attachments') + const { photos, videos } = attachmentParser(attachments) + + return { + author: { + username, + name: author.name, + profileImageUrl: author.avatar, + }, + createdAt, + text, + url, + entities: { + urls, + photos, + videos, + }, + } + } + + let context: BrowserContext | undefined + try { + const tweets: Tweet[] = [] + + context = await browser.createIncognitoBrowserContext() + const page = await context.newPage() + await page.goto(url, { + waitUntil: 'networkidle2', + timeout: 30000, // 30 seconds + }) + + const html = await page.content() + const document = parseHTML(html).document + + // get the main thread including tweets and threads + const mainThread = document.querySelector('.main-thread') + if (!mainThread) { + return [] + } + const timelineItems = Array.from( + mainThread.querySelectorAll('.timeline-item') + ) + for (let i = 0; i < timelineItems.length; i++) { + const item = timelineItems[i] + if (item.classList.contains('more-replies')) { + const newUrl = item.querySelector('a')?.getAttribute('href') + if (!newUrl) { + break + } + + // go to new url and wait for it to load + await page.goto(`${address}${newUrl}`, { + waitUntil: 'networkidle2', + timeout: 30000, // 30 seconds + }) + + const document = parseHTML(await page.content()).document + const nextThread = document.querySelector( + '.main-thread .after-tweet' + ) + if (!nextThread) { + break + } + + // get the new timeline items and add them to the list + const newTimelineItems = Array.from( + nextThread.querySelectorAll('.timeline-item') + ) + + timelineItems.push(...newTimelineItems) + continue + } + + const tweet = parseTweet(item) + tweet && tweets.push(tweet) + } + + return tweets + } catch (error) { + console.error('Error getting tweets', error) + + return [] + } finally { + if (context) { + await context.close() + } + } + } + + return genTweets(this.ADDRESS) + } + + parseTweetUrl = (url: string) => { + const match = url.match(this.URL_MATCH) + return { + domain: match?.[1], + username: match?.[4], + tweetId: match?.[5], + } + } + + titleForTweet = (author: { name: string }, text: string) => { + return `${author.name} on Twitter: ${truncate(text.replace(/http\S+/, ''), { + length: 100, + })}` + } + + formatTimestamp = (timestamp: string) => { + return DateTime.fromJSDate(new Date(timestamp)).toLocaleString( + DateTime.DATETIME_FULL + ) + } + + shouldPreHandle(url: string): boolean { + return this.URL_MATCH.test(url.toString()) + } + + async preHandle(url: string, browser: Browser): Promise { + const { tweetId, username, domain } = this.parseTweetUrl(url) + if (!tweetId || !username || !domain) { + throw new Error('could not parse tweet url') + } + const tweets = await this.getTweets(browser, username, tweetId) + + const tweet = tweets[0] + const author = tweet.author + // escape html entities in title + const title = this.titleForTweet(author, tweet.text) + const escapedTitle = _.escape(title) + const authorImage = `${this.ADDRESS}${author.profileImageUrl.replace( + '_normal', + '_400x400' + )}` + const description = _.escape(tweet.text) + + let tweetsContent = '' + for (const tweet of tweets) { + let text = tweet.text + if (tweet.entities && tweet.entities.urls) { + for (const urlObj of tweet.entities.urls) { + text = text.replace( + urlObj.url, + `${urlObj.display_url}` + ) + } + } + + const includesHtml = + tweet.entities.photos + ?.map( + (url) => + ` + + + + ` + ) + .join('\n') ?? '' + + tweetsContent += ` +

${text}

+ ${includesHtml} + ` + } + + const tweetUrl = ` + — ${ + author.username + } ${this.formatTimestamp(tweet.createdAt)}` + + const content = ` + + + + + + + + + + + +
+ ${tweetsContent} + ${tweetUrl} +
+ + ` + + return { content, url, title } + } +} diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index c1878e498..d80eaacf7 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -217,6 +217,12 @@ const getTweetIds = async ( context = await browser.createIncognitoBrowserContext() const page = await context.newPage() + // Modify this variable to control the size of viewport + const deviceScaleFactor = 0.2 + const height = Math.floor(2000 / deviceScaleFactor) + const width = Math.floor(1700 / deviceScaleFactor) + await page.setViewport({ width, height, deviceScaleFactor }) + await page.goto(pageURL, { waitUntil: 'networkidle0', timeout: 60000, // 60 seconds @@ -230,62 +236,48 @@ const getTweetIds = async ( const waitFor = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) - const ids: Set = new Set() + const ids = [] - const distance = 1080 - let scrollHeight = document.body.scrollHeight - let currentHeight = 0 - // keep scrolling until there are no more elements - while (currentHeight < scrollHeight) { - const timeNodes = Array.from(document.querySelectorAll('time')) + // Find the first Show thread button and click it + const showRepliesButton = Array.from( + document.querySelectorAll('div[dir]') + ) + .filter( + (node) => node.children[0] && node.children[0].tagName === 'SPAN' + ) + .find((node) => node.children[0].innerHTML === 'Show replies') - for (let i = 0; i < timeNodes.length; i++) { - const timeContainerAnchor: - | HTMLAnchorElement - | HTMLSpanElement - | null = timeNodes[i].parentElement - if (!timeContainerAnchor) continue + if (showRepliesButton) { + ;(showRepliesButton as HTMLElement).click() - if (timeContainerAnchor.tagName === 'SPAN') continue - - const href = timeContainerAnchor.getAttribute('href') - if (!href) continue - - // Get the tweet id and username from the href: https://twitter.com/username/status/1234567890 - const match = href.match(/\/([^/]+)\/status\/(\d+)/) - if (!match) continue - - const id = match[2] - const username = match[1] - - // stop at non-author replies - if (username !== author) return Array.from(ids) - ids.add(id) - } - - window.scrollBy(0, distance) - await waitFor(500) - currentHeight += distance - - // Find the show replies button and click it - if (currentHeight >= scrollHeight) { - const showRepliesButton = Array.from( - document.querySelectorAll('div[dir]') - ) - .filter( - (node) => node.children[0] && node.children[0].tagName === 'SPAN' - ) - .find((node) => node.children[0].innerHTML === 'Show replies') - - if (showRepliesButton) { - ;(showRepliesButton as HTMLElement).click() - await waitFor(1000) - scrollHeight = document.body.scrollHeight - } - } + await waitFor(2000) } - return Array.from(ids) + const timeNodes = Array.from(document.querySelectorAll('time')) + + for (const timeNode of timeNodes) { + /** @type {HTMLAnchorElement | HTMLSpanElement} */ + const timeContainerAnchor: HTMLAnchorElement | HTMLSpanElement | null = + timeNode.parentElement + if (!timeContainerAnchor) continue + + if (timeContainerAnchor.tagName === 'SPAN') continue + + const href = timeContainerAnchor.getAttribute('href') + if (!href) continue + + // Get the tweet id and username from the href: https://twitter.com/username/status/1234567890 + const match = href.match(/\/([^/]+)\/status\/(\d+)/) + if (!match) continue + + const id = match[2] + const username = match[1] + + // skip non-author replies + username === author && ids.push(id) + } + + return ids }, author)) as string[] } catch (error) { console.error('Error getting tweets', error)