mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
Merge pull request #2393 from omnivore-app/fix/scrape-twitter
fix/scrape twitter
This commit is contained in:
Hongbo Wu
GitHub
commit
01788fe0c9
3 changed files with 350 additions and 53 deletions
|
|
@ -27,12 +27,12 @@ import { DerstandardHandler } from './websites/derstandard-handler'
|
|||
import { GitHubHandler } from './websites/github-handler'
|
||||
import { ImageHandler } from './websites/image-handler'
|
||||
import { MediumHandler } from './websites/medium-handler'
|
||||
import { NitterHandler } from './websites/nitter-handler'
|
||||
import { PdfHandler } from './websites/pdf-handler'
|
||||
import { PipedVideoHandler } from './websites/piped-video-handler'
|
||||
import { ScrapingBeeHandler } from './websites/scrapingBee-handler'
|
||||
import { StackOverflowHandler } from './websites/stack-overflow-handler'
|
||||
import { TDotCoHandler } from './websites/t-dot-co-handler'
|
||||
import { TwitterHandler } from './websites/twitter-handler'
|
||||
import { WeixinQqHandler } from './websites/weixin-qq-handler'
|
||||
import { WikipediaHandler } from './websites/wikipedia-handler'
|
||||
import { YoutubeHandler } from './websites/youtube-handler'
|
||||
|
|
@ -64,7 +64,6 @@ const contentHandlers: ContentHandler[] = [
|
|||
new PdfHandler(),
|
||||
new ScrapingBeeHandler(),
|
||||
new TDotCoHandler(),
|
||||
new TwitterHandler(),
|
||||
new YoutubeHandler(),
|
||||
new WikipediaHandler(),
|
||||
new GitHubHandler(),
|
||||
|
|
@ -77,6 +76,7 @@ const contentHandlers: ContentHandler[] = [
|
|||
new EnergyWorldHandler(),
|
||||
new PipedVideoHandler(),
|
||||
new WeixinQqHandler(),
|
||||
new NitterHandler(),
|
||||
]
|
||||
|
||||
const newsletterHandlers: ContentHandler[] = [
|
||||
|
|
|
|||
305
packages/content-handler/src/websites/nitter-handler.ts
Normal file
305
packages/content-handler/src/websites/nitter-handler.ts
Normal file
|
|
@ -0,0 +1,305 @@
|
|||
import { parseHTML } from 'linkedom'
|
||||
import _, { truncate } from 'lodash'
|
||||
import { DateTime } from 'luxon'
|
||||
import { Browser, BrowserContext } from 'puppeteer-core'
|
||||
import { ContentHandler, PreHandleResult } from '../content-handler'
|
||||
|
||||
interface Tweet {
|
||||
url: string
|
||||
author: {
|
||||
username: string
|
||||
name: string
|
||||
profileImageUrl: string
|
||||
}
|
||||
text: string
|
||||
entities: {
|
||||
urls: {
|
||||
url: string
|
||||
display_url: string
|
||||
}[]
|
||||
photos: string[]
|
||||
videos: string[]
|
||||
}
|
||||
createdAt: string
|
||||
}
|
||||
|
||||
export class NitterHandler extends ContentHandler {
|
||||
// matches twitter.com and nitter.net urls
|
||||
URL_MATCH =
|
||||
/((twitter\.com)|(nitter\.net))\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
|
||||
ADDRESS = 'https://nitter.net'
|
||||
|
||||
constructor() {
|
||||
super()
|
||||
this.name = 'Nitter'
|
||||
}
|
||||
|
||||
async getTweets(browser: Browser, username: string, tweetId: string) {
|
||||
const url = `${this.ADDRESS}/${username}/status/${tweetId}`
|
||||
|
||||
async function genTweets(address: string): Promise<Tweet[]> {
|
||||
function authorParser(header: Element) {
|
||||
const avatar = header
|
||||
.querySelector('.tweet-avatar img')
|
||||
?.getAttribute('src')
|
||||
if (!avatar) {
|
||||
return null
|
||||
}
|
||||
const name = header.querySelector('.fullname')?.getAttribute('title')
|
||||
if (!name) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
avatar,
|
||||
name,
|
||||
}
|
||||
}
|
||||
|
||||
function dateParser(tweetDate: string) {
|
||||
const validDateTime = tweetDate.replace(' · ', ' ')
|
||||
|
||||
return new Date(validDateTime).toISOString()
|
||||
}
|
||||
|
||||
function attachmentParser(attachments: Element | null) {
|
||||
if (!attachments) return { photos: [], videos: [] }
|
||||
|
||||
const photos = Array.from(attachments.querySelectorAll('img')).map(
|
||||
(i) => i.getAttribute('src') ?? ''
|
||||
)
|
||||
const videos = Array.from(attachments.querySelectorAll('source')).map(
|
||||
(i) => i.getAttribute('src') ?? ''
|
||||
)
|
||||
return {
|
||||
photos,
|
||||
videos,
|
||||
}
|
||||
}
|
||||
|
||||
function parseTweet(tweet: Element): Tweet | null {
|
||||
const header = tweet.querySelector('.tweet-header')
|
||||
if (!header) {
|
||||
return null
|
||||
}
|
||||
const author = authorParser(header)
|
||||
if (!author) {
|
||||
return null
|
||||
}
|
||||
|
||||
const body = tweet.querySelector('.tweet-body')
|
||||
if (!body) {
|
||||
return null
|
||||
}
|
||||
|
||||
const tweetDate = body
|
||||
.querySelector('.tweet-date a')
|
||||
?.getAttribute('title')
|
||||
if (!tweetDate) {
|
||||
return null
|
||||
}
|
||||
const createdAt = dateParser(tweetDate)
|
||||
|
||||
const content = body.querySelector('.tweet-content')
|
||||
if (!content) {
|
||||
return null
|
||||
}
|
||||
const text = content.textContent ?? ''
|
||||
const urls = Array.from(content.querySelectorAll('a')).map((a) => ({
|
||||
url: a.getAttribute('href') ?? '',
|
||||
display_url: a.textContent ?? '',
|
||||
}))
|
||||
|
||||
const attachments = body.querySelector('.attachments')
|
||||
const { photos, videos } = attachmentParser(attachments)
|
||||
|
||||
return {
|
||||
author: {
|
||||
username,
|
||||
name: author.name,
|
||||
profileImageUrl: author.avatar,
|
||||
},
|
||||
createdAt,
|
||||
text,
|
||||
url,
|
||||
entities: {
|
||||
urls,
|
||||
photos,
|
||||
videos,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
let context: BrowserContext | undefined
|
||||
try {
|
||||
const tweets: Tweet[] = []
|
||||
|
||||
context = await browser.createIncognitoBrowserContext()
|
||||
const page = await context.newPage()
|
||||
await page.goto(url, {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 30000, // 30 seconds
|
||||
})
|
||||
|
||||
const html = await page.content()
|
||||
const document = parseHTML(html).document
|
||||
|
||||
// get the main thread including tweets and threads
|
||||
const mainThread = document.querySelector('.main-thread')
|
||||
if (!mainThread) {
|
||||
return []
|
||||
}
|
||||
const timelineItems = Array.from(
|
||||
mainThread.querySelectorAll('.timeline-item')
|
||||
)
|
||||
for (let i = 0; i < timelineItems.length; i++) {
|
||||
const item = timelineItems[i]
|
||||
if (item.classList.contains('more-replies')) {
|
||||
const newUrl = item.querySelector('a')?.getAttribute('href')
|
||||
if (!newUrl) {
|
||||
break
|
||||
}
|
||||
|
||||
// go to new url and wait for it to load
|
||||
await page.goto(`${address}${newUrl}`, {
|
||||
waitUntil: 'networkidle2',
|
||||
timeout: 30000, // 30 seconds
|
||||
})
|
||||
|
||||
const document = parseHTML(await page.content()).document
|
||||
const nextThread = document.querySelector(
|
||||
'.main-thread .after-tweet'
|
||||
)
|
||||
if (!nextThread) {
|
||||
break
|
||||
}
|
||||
|
||||
// get the new timeline items and add them to the list
|
||||
const newTimelineItems = Array.from(
|
||||
nextThread.querySelectorAll('.timeline-item')
|
||||
)
|
||||
|
||||
timelineItems.push(...newTimelineItems)
|
||||
continue
|
||||
}
|
||||
|
||||
const tweet = parseTweet(item)
|
||||
tweet && tweets.push(tweet)
|
||||
}
|
||||
|
||||
return tweets
|
||||
} catch (error) {
|
||||
console.error('Error getting tweets', error)
|
||||
|
||||
return []
|
||||
} finally {
|
||||
if (context) {
|
||||
await context.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return genTweets(this.ADDRESS)
|
||||
}
|
||||
|
||||
parseTweetUrl = (url: string) => {
|
||||
const match = url.match(this.URL_MATCH)
|
||||
return {
|
||||
domain: match?.[1],
|
||||
username: match?.[4],
|
||||
tweetId: match?.[5],
|
||||
}
|
||||
}
|
||||
|
||||
titleForTweet = (author: { name: string }, text: string) => {
|
||||
return `${author.name} on Twitter: ${truncate(text.replace(/http\S+/, ''), {
|
||||
length: 100,
|
||||
})}`
|
||||
}
|
||||
|
||||
formatTimestamp = (timestamp: string) => {
|
||||
return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(
|
||||
DateTime.DATETIME_FULL
|
||||
)
|
||||
}
|
||||
|
||||
shouldPreHandle(url: string): boolean {
|
||||
return this.URL_MATCH.test(url.toString())
|
||||
}
|
||||
|
||||
async preHandle(url: string, browser: Browser): Promise<PreHandleResult> {
|
||||
const { tweetId, username, domain } = this.parseTweetUrl(url)
|
||||
if (!tweetId || !username || !domain) {
|
||||
throw new Error('could not parse tweet url')
|
||||
}
|
||||
const tweets = await this.getTweets(browser, username, tweetId)
|
||||
|
||||
const tweet = tweets[0]
|
||||
const author = tweet.author
|
||||
// escape html entities in title
|
||||
const title = this.titleForTweet(author, tweet.text)
|
||||
const escapedTitle = _.escape(title)
|
||||
const authorImage = `${this.ADDRESS}${author.profileImageUrl.replace(
|
||||
'_normal',
|
||||
'_400x400'
|
||||
)}`
|
||||
const description = _.escape(tweet.text)
|
||||
|
||||
let tweetsContent = ''
|
||||
for (const tweet of tweets) {
|
||||
let text = tweet.text
|
||||
if (tweet.entities && tweet.entities.urls) {
|
||||
for (const urlObj of tweet.entities.urls) {
|
||||
text = text.replace(
|
||||
urlObj.url,
|
||||
`<a href="${urlObj.url}">${urlObj.display_url}</a>`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const includesHtml =
|
||||
tweet.entities.photos
|
||||
?.map(
|
||||
(url) =>
|
||||
`<a class="media-link" href=${this.ADDRESS}${url}>
|
||||
<picture>
|
||||
<img class="tweet-img" src=${this.ADDRESS}${url} />
|
||||
</picture>
|
||||
</a>`
|
||||
)
|
||||
.join('\n') ?? ''
|
||||
|
||||
tweetsContent += `
|
||||
<p>${text}</p>
|
||||
${includesHtml}
|
||||
`
|
||||
}
|
||||
|
||||
const tweetUrl = `
|
||||
— <a href="https://${domain}/${author.username}">${
|
||||
author.username
|
||||
}</a> <span itemscope itemtype="https://schema.org/Person" itemprop="author">${
|
||||
author.name
|
||||
}</span> <a href="${url}">${this.formatTimestamp(tweet.createdAt)}</a>`
|
||||
|
||||
const content = `
|
||||
<html>
|
||||
<head>
|
||||
<meta property="og:image" content="${authorImage}" />
|
||||
<meta property="og:image:secure_url" content="${authorImage}" />
|
||||
<meta property="og:title" content="${escapedTitle}" />
|
||||
<meta property="og:description" content="${description}" />
|
||||
<meta property="article:published_time" content="${tweet.createdAt}" />
|
||||
<meta property="og:site_name" content="Twitter" />
|
||||
<meta property="og:type" content="tweet" />
|
||||
</head>
|
||||
<body>
|
||||
<div>
|
||||
${tweetsContent}
|
||||
${tweetUrl}
|
||||
</div>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
return { content, url, title }
|
||||
}
|
||||
}
|
||||
|
|
@ -217,6 +217,12 @@ const getTweetIds = async (
|
|||
context = await browser.createIncognitoBrowserContext()
|
||||
const page = await context.newPage()
|
||||
|
||||
// Modify this variable to control the size of viewport
|
||||
const deviceScaleFactor = 0.2
|
||||
const height = Math.floor(2000 / deviceScaleFactor)
|
||||
const width = Math.floor(1700 / deviceScaleFactor)
|
||||
await page.setViewport({ width, height, deviceScaleFactor })
|
||||
|
||||
await page.goto(pageURL, {
|
||||
waitUntil: 'networkidle0',
|
||||
timeout: 60000, // 60 seconds
|
||||
|
|
@ -230,62 +236,48 @@ const getTweetIds = async (
|
|||
const waitFor = (ms: number) =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms))
|
||||
|
||||
const ids: Set<string> = new Set()
|
||||
const ids = []
|
||||
|
||||
const distance = 1080
|
||||
let scrollHeight = document.body.scrollHeight
|
||||
let currentHeight = 0
|
||||
// keep scrolling until there are no more elements
|
||||
while (currentHeight < scrollHeight) {
|
||||
const timeNodes = Array.from(document.querySelectorAll('time'))
|
||||
// Find the first Show thread button and click it
|
||||
const showRepliesButton = Array.from(
|
||||
document.querySelectorAll('div[dir]')
|
||||
)
|
||||
.filter(
|
||||
(node) => node.children[0] && node.children[0].tagName === 'SPAN'
|
||||
)
|
||||
.find((node) => node.children[0].innerHTML === 'Show replies')
|
||||
|
||||
for (let i = 0; i < timeNodes.length; i++) {
|
||||
const timeContainerAnchor:
|
||||
| HTMLAnchorElement
|
||||
| HTMLSpanElement
|
||||
| null = timeNodes[i].parentElement
|
||||
if (!timeContainerAnchor) continue
|
||||
if (showRepliesButton) {
|
||||
;(showRepliesButton as HTMLElement).click()
|
||||
|
||||
if (timeContainerAnchor.tagName === 'SPAN') continue
|
||||
|
||||
const href = timeContainerAnchor.getAttribute('href')
|
||||
if (!href) continue
|
||||
|
||||
// Get the tweet id and username from the href: https://twitter.com/username/status/1234567890
|
||||
const match = href.match(/\/([^/]+)\/status\/(\d+)/)
|
||||
if (!match) continue
|
||||
|
||||
const id = match[2]
|
||||
const username = match[1]
|
||||
|
||||
// stop at non-author replies
|
||||
if (username !== author) return Array.from(ids)
|
||||
ids.add(id)
|
||||
}
|
||||
|
||||
window.scrollBy(0, distance)
|
||||
await waitFor(500)
|
||||
currentHeight += distance
|
||||
|
||||
// Find the show replies button and click it
|
||||
if (currentHeight >= scrollHeight) {
|
||||
const showRepliesButton = Array.from(
|
||||
document.querySelectorAll('div[dir]')
|
||||
)
|
||||
.filter(
|
||||
(node) => node.children[0] && node.children[0].tagName === 'SPAN'
|
||||
)
|
||||
.find((node) => node.children[0].innerHTML === 'Show replies')
|
||||
|
||||
if (showRepliesButton) {
|
||||
;(showRepliesButton as HTMLElement).click()
|
||||
await waitFor(1000)
|
||||
scrollHeight = document.body.scrollHeight
|
||||
}
|
||||
}
|
||||
await waitFor(2000)
|
||||
}
|
||||
|
||||
return Array.from(ids)
|
||||
const timeNodes = Array.from(document.querySelectorAll('time'))
|
||||
|
||||
for (const timeNode of timeNodes) {
|
||||
/** @type {HTMLAnchorElement | HTMLSpanElement} */
|
||||
const timeContainerAnchor: HTMLAnchorElement | HTMLSpanElement | null =
|
||||
timeNode.parentElement
|
||||
if (!timeContainerAnchor) continue
|
||||
|
||||
if (timeContainerAnchor.tagName === 'SPAN') continue
|
||||
|
||||
const href = timeContainerAnchor.getAttribute('href')
|
||||
if (!href) continue
|
||||
|
||||
// Get the tweet id and username from the href: https://twitter.com/username/status/1234567890
|
||||
const match = href.match(/\/([^/]+)\/status\/(\d+)/)
|
||||
if (!match) continue
|
||||
|
||||
const id = match[2]
|
||||
const username = match[1]
|
||||
|
||||
// skip non-author replies
|
||||
username === author && ids.push(id)
|
||||
}
|
||||
|
||||
return ids
|
||||
}, author)) as string[]
|
||||
} catch (error) {
|
||||
console.error('Error getting tweets', error)
|
||||
|
|
|
|||
Loading…
Reference in a new issue