diff --git a/packages/content-handler/package.json b/packages/content-handler/package.json index 5cbda3c86..bae0e5953 100644 --- a/packages/content-handler/package.json +++ b/packages/content-handler/package.json @@ -16,5 +16,11 @@ }, "devDependencies": { "eslint-plugin-prettier": "^4.0.0" + }, + "dependencies": { + "axios": "^0.27.2", + "linkedom": "^0.14.16", + "luxon": "^3.0.4", + "underscore": "^1.13.6" } } diff --git a/packages/content-handler/src/handlers/apple-news-handler.ts b/packages/content-handler/src/handlers/apple-news-handler.ts new file mode 100644 index 000000000..a53c8ef02 --- /dev/null +++ b/packages/content-handler/src/handlers/apple-news-handler.ts @@ -0,0 +1,26 @@ +import { ContentHandler, PreHandleResult } from '../index' +import axios from 'axios' +import { parseHTML } from 'linkedom' + +export class AppleNewsHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + const u = new URL(url) + return u.hostname === 'apple.news' + } + + async preHandle(url: string, _document: Document): Promise { + const MOBILE_USER_AGENT = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36' + const response = await axios.get(url, { + headers: { 'User-Agent': MOBILE_USER_AGENT }, + }) + const data = response.data + const dom = parseHTML(data).document + // make sure it's a valid URL by wrapping in new URL + const href = dom + .querySelector('span.click-here') + ?.parentElement?.getAttribute('href') + const u = href ? new URL(href) : undefined + return { url: u?.href } + } +} diff --git a/packages/content-handler/src/handlers/bloomberg-handler.ts b/packages/content-handler/src/handlers/bloomberg-handler.ts new file mode 100644 index 000000000..05100429c --- /dev/null +++ b/packages/content-handler/src/handlers/bloomberg-handler.ts @@ -0,0 +1,36 @@ +import { ContentHandler, PreHandleResult } from '../index' +import axios from 'axios' +import { parseHTML } from 'linkedom' + +class BloombergHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + const BLOOMBERG_URL_MATCH = + /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/ + return BLOOMBERG_URL_MATCH.test(url.toString()) + } + + async preHandle(url: string, _document: Document): Promise { + console.log('prehandling bloomberg url', url) + + try { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + api_key: process.env.SCRAPINGBEE_API_KEY, + url: url, + return_page_source: true, + block_ads: true, + block_resources: false, + }, + }) + const dom = parseHTML(response.data).document + return { + title: dom.title, + content: dom.querySelector('body')?.innerHTML, + url: url, + } + } catch (error) { + console.error('error prehandling bloomberg url', error) + throw error + } + } +} diff --git a/packages/content-handler/src/handlers/derstandard-handler.ts b/packages/content-handler/src/handlers/derstandard-handler.ts new file mode 100644 index 000000000..fd77ce108 --- /dev/null +++ b/packages/content-handler/src/handlers/derstandard-handler.ts @@ -0,0 +1,29 @@ +import { ContentHandler, PreHandleResult } from '../index' +import axios from 'axios' +import { parseHTML } from 'linkedom' + +class DerstandardHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + const u = new URL(url) + return u.hostname === 'www.derstandard.at' + } + + async preHandle(url: string, _document: Document): Promise { + const response = await axios.get(url, { + // set cookie to give consent to get the article + headers: { + cookie: `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`, + }, + }) + const content = response.data + + const dom = parseHTML(content).document + const titleElement = dom.querySelector('.article-title') + titleElement && titleElement.remove() + + return { + content: dom.body.outerHTML, + title: titleElement?.textContent || undefined, + } + } +} diff --git a/packages/content-handler/src/handlers/image-handler.ts b/packages/content-handler/src/handlers/image-handler.ts new file mode 100644 index 000000000..3f994332b --- /dev/null +++ b/packages/content-handler/src/handlers/image-handler.ts @@ -0,0 +1,27 @@ +import { ContentHandler, PreHandleResult } from '../index' + +class ImageHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + const IMAGE_URL_PATTERN = /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i + return IMAGE_URL_PATTERN.test(url.toString()) + } + + async preHandle(url: string, _document: Document): Promise { + const title = url.toString().split('/').pop() + const content = ` + + + ${title} + + + + +
+ ${title} +
+ + ` + + return { title, content } + } +} diff --git a/packages/content-handler/src/handlers/medium-handler.ts b/packages/content-handler/src/handlers/medium-handler.ts new file mode 100644 index 000000000..d87d4338f --- /dev/null +++ b/packages/content-handler/src/handlers/medium-handler.ts @@ -0,0 +1,21 @@ +import { ContentHandler, PreHandleResult } from '../index' + +class MediumHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + const u = new URL(url) + return u.hostname.endsWith('medium.com') + } + + async preHandle(url: string, _document: Document): Promise { + console.log('prehandling medium url', url) + + try { + const res = new URL(url) + res.searchParams.delete('source') + return { url: res.toString() } + } catch (error) { + console.error('error prehandling medium url', error) + throw error + } + } +} diff --git a/packages/content-handler/src/handlers/pdf-handler.ts b/packages/content-handler/src/handlers/pdf-handler.ts new file mode 100644 index 000000000..54c9e6bf4 --- /dev/null +++ b/packages/content-handler/src/handlers/pdf-handler.ts @@ -0,0 +1,13 @@ +import { ContentHandler, PreHandleResult } from '../index' + +class PdfHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + const u = new URL(url) + const path = u.pathname.replace(u.search, '') + return path.endsWith('.pdf') + } + + async preHandle(_url: string, _document: Document): Promise { + return { contentType: 'application/pdf' } + } +} diff --git a/packages/content-handler/src/handlers/scrapingBee-handler.ts b/packages/content-handler/src/handlers/scrapingBee-handler.ts new file mode 100644 index 000000000..1c84ba06b --- /dev/null +++ b/packages/content-handler/src/handlers/scrapingBee-handler.ts @@ -0,0 +1,33 @@ +import { ContentHandler, PreHandleResult } from '../index' +import axios from 'axios' +import { parseHTML } from 'linkedom' + +class ScrapingBeeHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + const u = new URL(url) + const hostnames = ['nytimes.com', 'news.google.com'] + + return hostnames.some((h) => u.hostname.endsWith(h)) + } + + async preHandle(url: string, _document: Document): Promise { + console.log('prehandling url with scrapingbee', url) + + try { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + api_key: process.env.SCRAPINGBEE_API_KEY, + url: url, + return_page_source: true, + block_ads: true, + block_resources: false, + }, + }) + const dom = parseHTML(response.data).document + return { title: dom.title, content: response.data, url: url } + } catch (error) { + console.error('error prehandling url w/scrapingbee', error) + throw error + } + } +} diff --git a/packages/content-handler/src/handlers/t-dot-co-handler.ts b/packages/content-handler/src/handlers/t-dot-co-handler.ts new file mode 100644 index 000000000..e83443518 --- /dev/null +++ b/packages/content-handler/src/handlers/t-dot-co-handler.ts @@ -0,0 +1,21 @@ +import { ContentHandler } from '../index' +import axios from 'axios' + +class TDotCoHandler extends ContentHandler { + shouldResolve(url: string): boolean { + const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/ + return T_DOT_CO_URL_MATCH.test(url) + } + + async resolve(url: string) { + return await axios + .get(url, { maxRedirects: 0, validateStatus: null }) + .then((res) => { + return new URL(res.headers.location).href + }) + .catch((err) => { + console.log('err with t.co url', err) + return undefined + }) + } +} diff --git a/packages/content-handler/src/handlers/twitter-handler.ts b/packages/content-handler/src/handlers/twitter-handler.ts new file mode 100644 index 000000000..265bfe955 --- /dev/null +++ b/packages/content-handler/src/handlers/twitter-handler.ts @@ -0,0 +1,142 @@ +import { ContentHandler, PreHandleResult } from '../index' +import axios from 'axios' +import { DateTime } from 'luxon' +import _ from 'underscore' + +const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN +const TWITTER_URL_MATCH = + /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ + +const getTweetFields = () => { + const TWEET_FIELDS = + '&tweet.fields=attachments,author_id,conversation_id,created_at,' + + 'entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets,' + + 'source,withheld' + const EXPANSIONS = '&expansions=author_id,attachments.media_keys' + const USER_FIELDS = + '&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld' + const MEDIA_FIELDS = + '&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width' + + return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}` +} + +const getTweetById = async (id: string) => { + const BASE_ENDPOINT = 'https://api.twitter.com/2/tweets/' + const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields()) + + return await axios.get(apiUrl.toString(), { + headers: { + Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, + redirect: 'follow', + }, + }) +} + +const titleForAuthor = (author: any) => { + return `${author.name} on Twitter` +} + +const tweetIdFromStatusUrl = (url: string): string | undefined => { + const match = url.toString().match(TWITTER_URL_MATCH) + return match?.[2] +} + +const formatTimestamp = (timestamp: string) => { + return DateTime.fromJSDate(new Date(timestamp)).toLocaleString( + DateTime.DATETIME_FULL + ) +} + +class TwitterHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + return !!TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) + } + + // version of the handler that uses the oembed API + // This isn't great as it doesn't work well with our + // readability API. But could potentially give a more consistent + // look to the tweets + // prehandle: async (url, env) => { + // const oeTweet = await embeddedTweet(url) + // const dom = new JSDOM(oeTweet.data.html); + // const bq = dom.window.document.querySelector('blockquote') + // console.log('blockquote:', bq); + + // const title = titleForTweet(oeTweet) + // return { title, content: '
' + bq.innerHTML + '
', url: oeTweet.data.url }; + // } + + async preHandle(url: string, _document: Document): Promise { + console.log('prehandling twitter url', url) + + const tweetId = tweetIdFromStatusUrl(url) + if (!tweetId) { + throw new Error('could not find tweet id in url') + } + const tweetData = (await getTweetById(tweetId)).data + const authorId = tweetData.data.author_id + const author = tweetData.includes.users.filter( + (u: any) => (u.id = authorId) + )[0] + // escape html entities in title + const title = _.escape(titleForAuthor(author)) + const authorImage = author.profile_image_url.replace('_normal', '_400x400') + + let text = tweetData.data.text + if (tweetData.data.entities && tweetData.data.entities.urls) { + for (let urlObj of tweetData.data.entities.urls) { + text = text.replace( + urlObj.url, + `${urlObj.display_url}` + ) + } + } + + const front = ` +
+

${text}

+ ` + + var includesHtml = '' + if (tweetData.includes.media) { + includesHtml = tweetData.includes.media + .map((m: any) => { + const linkUrl = m.type == 'photo' ? m.url : url + const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url + const mediaOpen = ` + + + + ` + return mediaOpen + }) + .join('\n') + } + + const back = ` + — ${ + author.username + } ${author.name} ${formatTimestamp( + tweetData.data.created_at + )} +
+ ` + const content = ` + + + + + + + + ${front} + ${includesHtml} + ${back} + ` + + return { content, url, title } + } +} diff --git a/packages/content-handler/src/handlers/youtube-handler.ts b/packages/content-handler/src/handlers/youtube-handler.ts new file mode 100644 index 000000000..60ae22970 --- /dev/null +++ b/packages/content-handler/src/handlers/youtube-handler.ts @@ -0,0 +1,66 @@ +import { PreHandleResult } from '../index' +import axios from 'axios' +import _ from 'underscore' + +const { ContentHandler } = require('../index') + +const YOUTUBE_URL_MATCH = + /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/ + +function getYoutubeVideoId(url: string) { + const u = new URL(url) + const videoId = u.searchParams.get('v') + if (!videoId) { + const match = url.toString().match(YOUTUBE_URL_MATCH) + if (match === null || match.length < 6 || !match[5]) { + return undefined + } + return match[5] + } + return videoId +} + +class YoutubeHandler extends ContentHandler { + shouldPreHandle(url: string, _dom: Document): boolean { + return YOUTUBE_URL_MATCH.test(url.toString()) + } + + async preHandle(url: string, _document: Document): Promise { + const videoId = getYoutubeVideoId(url) + if (!videoId) { + return {} + } + + const oembedUrl = + `https://www.youtube.com/oembed?format=json&url=` + + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`) + const oembed = (await axios.get(oembedUrl.toString())).data + // escape html entities in title + const title = _.escape(oembed.title) + const ratio = oembed.width / oembed.height + const thumbnail = oembed.thumbnail_url + const height = 350 + const width = height * ratio + const authorName = _.escape(oembed.author_name) + + const content = ` + + ${title} + + + + + + + + +

${title}

+ + + ` + + console.log('got video id', videoId) + + return { content, title: 'Youtube Content' } + } +} diff --git a/packages/content-handler/src/index.ts b/packages/content-handler/src/index.ts index 24f3e874e..8f3cfbd81 100644 --- a/packages/content-handler/src/index.ts +++ b/packages/content-handler/src/index.ts @@ -17,18 +17,26 @@ interface NewsletterMessage { unsubHttpUrl?: string } +export interface PreHandleResult { + url?: string + title?: string + content?: string + contentType?: string + dom?: Document +} + export class ContentHandler { protected senderRegex = /NEWSLETTER_SENDER_REGEX/ protected urlRegex = /NEWSLETTER_URL_REGEX/ protected defaultUrl = 'NEWSLETTER_DEFAULT_URL' protected name = '' - shouldPrehandle(url: URL, dom: Document): boolean { + shouldPreHandle(url: string, dom: Document): boolean { return false } - prehandle(url: URL, document: Document): Promise { - return Promise.resolve(document) + preHandle(url: string, document: Document): Promise { + return Promise.resolve({ url, dom: document }) } isNewsletter(postHeader: string, from: string, unSubHeader: string): boolean { @@ -67,14 +75,14 @@ export class ContentHandler { } } - async handleNewsletter( + handleNewsletter( email: string, html: string, postHeader: string, title: string, from: string, unSubHeader: string - ): Promise { + ): NewsletterMessage { console.log('handleNewsletter', email, postHeader, title, from) if (!email || !html || !title || !from) { diff --git a/yarn.lock b/yarn.lock index 97a48bba4..3da81da9b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -17652,6 +17652,17 @@ linkedom@^0.14.12: htmlparser2 "^8.0.1" uhyphen "^0.1.0" +linkedom@^0.14.16: + version "0.14.16" + resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.16.tgz#124eb006fad1dfe7ed8f96ec8ae74ab0fb0fd88e" + integrity sha512-a4QWl4W93P15/x+4d9k8K+C81nOzQeGOs3D37uG0TFqKZYGLEyZwXweSFrypK8yvUx5U2cuZKkdDIOjaouv3ag== + dependencies: + css-select "^5.1.0" + cssom "^0.5.0" + html-escaper "^3.0.3" + htmlparser2 "^8.0.1" + uhyphen "^0.1.0" + linkedom@^0.14.9: version "0.14.9" resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.14.9.tgz#34c6f15eddc809406f42d8ee48cd30b0222eccb0" @@ -18151,6 +18162,11 @@ luxon@^2.3.1: resolved "https://registry.yarnpkg.com/luxon/-/luxon-2.3.1.tgz#f276b1b53fd9a740a60e666a541a7f6dbed4155a" integrity sha512-I8vnjOmhXsMSlNMZlMkSOvgrxKJl0uOsEzdGgGNZuZPaS9KlefpE9KV95QFftlJSC+1UyCC9/I69R02cz/zcCA== +luxon@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.0.4.tgz#d179e4e9f05e092241e7044f64aaa54796b03929" + integrity sha512-aV48rGUwP/Vydn8HT+5cdr26YYQiUZ42NM6ToMoaGKwYfWbfLeRkEu1wXWMHBZT6+KyLfcbbtVcoQFCbbPjKlw== + lz-string@^1.4.4: version "1.4.4" resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26" @@ -24493,6 +24509,11 @@ underscore@^1.13.4, underscore@^1.9.1: resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee" integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ== +underscore@^1.13.6: + version "1.13.6" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.6.tgz#04786a1f589dc6c09f761fc5f45b89e935136441" + integrity sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A== + undici@^4.9.3: version "4.14.1" resolved "https://registry.yarnpkg.com/undici/-/undici-4.14.1.tgz#7633b143a8a10d6d63335e00511d071e8d52a1d9"