From fd934a949ed3b4def892e2a553fb97801261de79 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Sun, 8 May 2022 21:52:36 -0700 Subject: [PATCH] Add a standalone service content fetching --- packages/content-fetch/Dockerfile | 41 ++ packages/content-fetch/app.js | 26 + packages/content-fetch/apple-news-handler.js | 39 ++ packages/content-fetch/bloomberg-handler.js | 40 ++ packages/content-fetch/derstandard-handler.js | 36 ++ packages/content-fetch/fetch-content.js | 585 ++++++++++++++++++ packages/content-fetch/image-handler.js | 34 + packages/content-fetch/medium-handler.js | 33 + packages/content-fetch/package.json | 24 + packages/content-fetch/pdf-handler.js | 21 + packages/content-fetch/t-dot-co-handler.js | 32 + .../test/apple-news-handler.test.js | 9 + .../test/youtube-handler.test.js | 12 + packages/content-fetch/twitter-handler.js | 170 +++++ packages/content-fetch/youtube-handler.js | 68 ++ 15 files changed, 1170 insertions(+) create mode 100644 packages/content-fetch/Dockerfile create mode 100644 packages/content-fetch/app.js create mode 100644 packages/content-fetch/apple-news-handler.js create mode 100644 packages/content-fetch/bloomberg-handler.js create mode 100644 packages/content-fetch/derstandard-handler.js create mode 100644 packages/content-fetch/fetch-content.js create mode 100644 packages/content-fetch/image-handler.js create mode 100644 packages/content-fetch/medium-handler.js create mode 100644 packages/content-fetch/package.json create mode 100644 packages/content-fetch/pdf-handler.js create mode 100644 packages/content-fetch/t-dot-co-handler.js create mode 100644 packages/content-fetch/test/apple-news-handler.test.js create mode 100644 packages/content-fetch/test/youtube-handler.test.js create mode 100644 packages/content-fetch/twitter-handler.js create mode 100644 packages/content-fetch/youtube-handler.js diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile new file mode 100644 index 000000000..dd687e512 --- /dev/null +++ b/packages/content-fetch/Dockerfile @@ -0,0 +1,41 @@ +FROM node:14.18-alpine + +# Installs latest Chromium (92) package. +RUN apk add --no-cache \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont \ + nodejs \ + yarn + +# Tell Puppeteer to skip installing Chrome. We'll be using the installed package. +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + +# Puppeteer v10.0.0 works with Chromium 92. +RUN yarn add puppeteer@10.0.0 + +# Add user so we don't need --no-sandbox. +RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ + && mkdir -p /home/pptruser/Downloads /app \ + && chown -R pptruser:pptruser /home/pptruser \ + && chown -R pptruser:pptruser /app + +# Run everything after as non-privileged user. +WORKDIR /app + +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +ENV CHROMIUM_PATH /usr/bin/chromium-browser +ENV LAUNCH_HEADLESS=true + +COPY . /app/ +WORKDIR app + +RUN yarn install --pure-lockfile + +EXPOSE 8080 + +ENTRYPOINT ["yarn", "start"] \ No newline at end of file diff --git a/packages/content-fetch/app.js b/packages/content-fetch/app.js new file mode 100644 index 000000000..c0b6b5bf6 --- /dev/null +++ b/packages/content-fetch/app.js @@ -0,0 +1,26 @@ + + +const express = require('express'); + +const app = express(); +const fetchContent = require('./fetch-content'); + +app.use(express.json()); +app.use(express.urlencoded({ extended: true })); + + +app.get('/', (req, res) => { + fetchContent(req, res) +}); + +app.post('/', (req, res) => { + fetchContent(req, res) +}); + +const PORT = parseInt(process.env.PORT) || 8080; +app.listen(PORT, () => { + console.log(`App listening on port ${PORT}`); + console.log('Press Ctrl+C to quit.'); +}); + +module.exports = app; \ No newline at end of file diff --git a/packages/content-fetch/apple-news-handler.js b/packages/content-fetch/apple-news-handler.js new file mode 100644 index 000000000..ccdfd62ab --- /dev/null +++ b/packages/content-fetch/apple-news-handler.js @@ -0,0 +1,39 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const Url = require('url'); +const axios = require('axios'); +const { promisify } = require('util'); +const { DateTime } = require('luxon'); +const os = require('os'); +const jsdom = require("jsdom"); +const { Cipher } = require('crypto'); +const { JSDOM } = jsdom; + + + +exports.appleNewsHandler = { + + shouldPrehandle: (url, env) => { + const u = new URL(url); + if (u.hostname === 'apple.news') { + return true; + } + return false + }, + + prehandle: async (url, env) => { + const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36' + const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT } } ); + const data = response.data; + + const dom = new JSDOM(data); + + // make sure its a valid URL by wrapping in new URL + const u = new URL(dom.window.document.querySelector('span.click-here').parentNode.href); + return { url: u.href }; + } +} diff --git a/packages/content-fetch/bloomberg-handler.js b/packages/content-fetch/bloomberg-handler.js new file mode 100644 index 000000000..85a467aa1 --- /dev/null +++ b/packages/content-fetch/bloomberg-handler.js @@ -0,0 +1,40 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const axios = require('axios'); +const os = require('os'); +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; + +exports.bloombergHandler = { + + shouldPrehandle: (url, env) => { + const BLOOMBERG_URL_MATCH = + /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/ + return BLOOMBERG_URL_MATCH.test(url.toString()) + }, + + prehandle: async (url, env) => { + console.log('prehandling bloomberg url', url) + + try { + const response = await axios.get('https://app.scrapingbee.com/api/v1', { + params: { + 'api_key': process.env.SCRAPINGBEE_API_KEY, + 'url': url, + 'return_page_source': true, + 'block_ads': true, + 'block_resources': false, + } + }) + const dom = new JSDOM(response.data); + return { title: dom.window.document.title, content: dom.window.document.querySelector('body').innerHTML, url: url } + } catch (error) { + console.error('error prehandling bloomberg url', error) + throw error + } + } +} diff --git a/packages/content-fetch/derstandard-handler.js b/packages/content-fetch/derstandard-handler.js new file mode 100644 index 000000000..d9839e7c9 --- /dev/null +++ b/packages/content-fetch/derstandard-handler.js @@ -0,0 +1,36 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const axios = require('axios'); +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; + +exports.derstandardHandler = { + shouldPrehandle: (url, env) => { + const u = new URL(url); + return u.hostname === 'www.derstandard.at'; + }, + + prehandle: async (url, env) => { + const response = await axios.get(url, { + // set cookie to give consent to get the article + headers: { + 'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6` + }, + }); + const content = response.data; + + var title = undefined + const dom = new JSDOM(content) + const titleElement = dom.window.document.querySelector('.article-title') + if (!titleElement) { + title = titleElement.textContent + titleElement.remove() + } + + return { content: dom.window.document.body.outerHTML, title: title }; + } +} diff --git a/packages/content-fetch/fetch-content.js b/packages/content-fetch/fetch-content.js new file mode 100644 index 000000000..b40e03445 --- /dev/null +++ b/packages/content-fetch/fetch-content.js @@ -0,0 +1,585 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const Url = require('url'); +const puppeteer = require('puppeteer-extra'); +const axios = require('axios'); +const jwt = require('jsonwebtoken'); +const { promisify } = require('util'); +const signToken = promisify(jwt.sign); +const { appleNewsHandler } = require('./apple-news-handler'); +const { twitterHandler } = require('./twitter-handler'); +const { youtubeHandler } = require('./youtube-handler'); +const { tDotCoHandler } = require('./t-dot-co-handler'); +const { pdfHandler } = require('./pdf-handler'); +const { mediumHandler } = require('./medium-handler'); +const { derstandardHandler } = require('./derstandard-handler'); +const { imageHandler } = require('./image-handler'); + +const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' +const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' +const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] + +const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; + +// Add stealth plugin and use defaults (all tricks to hide puppeteer usage) +// const StealthPlugin = require('puppeteer-extra-plugin-stealth'); +// puppeteer.use(StealthPlugin()); + +const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker') +puppeteer.use(AdblockerPlugin({blockTrackers: true})) + + +const userAgentForUrl = (url) => { + try { + const u = new URL(url); + for (const host of NON_BOT_HOSTS) { + if (u.hostname.endsWith(host)) { + return NON_BOT_DESKTOP_USER_AGENT; + } + } + } catch (e) { + console.log('error getting user agent for url', url, e) + } + return DESKTOP_USER_AGENT +}; + +// launch Puppeteer +const getBrowserPromise = (async () => { + return puppeteer.launch({ + args: ['--no-sandbox'], + defaultViewport: { height: 1080, width: 1920 }, + executablePath: process.env.CHROMIUM_PATH , + headless: true, // process.env.LAUNCH_HEADLESS ? true : false, + timeout: 0, + }); +})(); + +let logRecord, functionStartTime; + +const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => { + const stream = await axios.get(contentObjUrl, { responseType: 'stream' }); + return await axios.put(uploadSignedUrl, stream.data, { + headers: { + 'Content-Type': contentType, + }, + maxBodyLength: 1000000000, + maxContentLength: 100000000, + }) +}; + +const getUploadIdAndSignedUrl = async (userId, url) => { + const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); + const data = JSON.stringify({ + query: `mutation UploadFileRequest($input: UploadFileRequestInput!) { + uploadFileRequest(input:$input) { + ... on UploadFileRequestError { + errorCodes + } + ... on UploadFileRequestSuccess { + id + uploadSignedUrl + } + } + }`, + variables: { + input: { + url, + contentType: 'application/pdf', + } + } + }); + + const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); + return response.data.data.uploadFileRequest; +}; + +const uploadPdf = async (url, userId) => { + validateUrlString(url); + + const uploadResult = await getUploadIdAndSignedUrl(userId, url); + await uploadToSignedUrl(uploadResult, 'application/pdf', url); + return uploadResult.id; +}; + +const sendCreateArticleMutation = async (userId, input) => { + const data = JSON.stringify({ + query: `mutation CreateArticle ($input: CreateArticleInput!){ + createArticle(input:$input){ + ... on CreateArticleSuccess{ + createdArticle{ + id + } + } + ... on CreateArticleError{ + errorCodes + } + } + }`, + variables: { + input: Object.assign({}, input , { source: 'puppeteer-parse' }), + }, + }); + + const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); + const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); + console.log('response', response); + return response.data.data.createArticle; +}; + +const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => { + return sendCreateArticleMutation(userId, { + url: encodeURI(url), + articleSavingRequestId, + uploadFileId: uploadFileId, + }, + ); +}; + +const handlers = { + 'pdf': pdfHandler, + 'apple-news': appleNewsHandler, + 'twitter': twitterHandler, + 'youtube': youtubeHandler, + 't-dot-co': tDotCoHandler, + 'medium': mediumHandler, + 'derstandard': derstandardHandler, + 'image': imageHandler, +}; + + +async function fetchContent(req, res) { + functionStartTime = Date.now(); + + let url = getUrl(req); + const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined); + const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); + + console.log('user id', userId, 'url', url) + + logRecord = { + url, + userId, + articleSavingRequestId, + labels: { + source: 'parseContent', + }, + }; + + console.log(`Article parsing request`, logRecord); + + if (!url) { + logRecord.urlIsInvalid = true; + console.log(`Valid URL to parse not specified`, logRecord); + return res.sendStatus(400); + } + + // if (!userId || !articleSavingRequestId) { + // Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query }); + // console.log(`Invalid parameters`, logRecord); + // return res.sendStatus(400); + // } + + // Before we run the regular handlers we check to see if we need tp + // pre-resolve the URL. TODO: This should probably happen recursively, + // so URLs can be pre-resolved, handled, pre-resolved, handled, etc. + for (const [key, handler] of Object.entries(handlers)) { + if (handler.shouldResolve && handler.shouldResolve(url)) { + try { + url = await handler.resolve(url); + validateUrlString(url); + } catch (err) { + console.log('error resolving url with handler', key, err); + } + break; + } + } + + // Before we fetch the page we check the handlers, to see if they want + // to perform a prefetch action that can modify our requests. + // enumerate the handlers and see if any of them want to handle the request + const handler = Object.keys(handlers).find(key => { + try { + return handlers[key].shouldPrehandle(url) + } catch (e) { + console.log('error with handler: ', key, e); + } + return false; + }); + + var title = undefined; + var content = undefined; + var contentType = undefined; + + if (handler) { + try { + // The only handler we have now can modify the URL, but in the + // future maybe we let it modify content. In that case + // we might exit the request early. + console.log('pre-handling url with handler: ', handler); + + const result = await handlers[handler].prehandle(url); + if (result && result.url) { + url = result.url + validateUrlString(url); + } + if (result && result.title) { title = result.title } + if (result && result.content) { content = result.content } + if (result && result.contentType) { contentType = result.contentType } + } catch (e) { + console.log('error with handler: ', handler, e); + } + } + + var context, page, finalUrl; + if ((!content || !title) && contentType !== 'application/pdf') { + const result = await retrievePage(url) + if (result && result.context) { context = result.context } + if (result && result.page) { page = result.page } + if (result && result.finalUrl) { finalUrl = result.finalUrl } + if (result && result.contentType) { contentType = result.contentType } + } else { + finalUrl = url + } + + try { + if (contentType === 'application/pdf') { + const uploadedFileId = await uploadPdf(finalUrl, userId); + const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId); + } else { + if (!content || !title) { + const result = await retrieveHtml(page); + title = result.title; + content = result.domContent; + } else { + console.log('using prefetched content and title'); + console.log(content); + } + + logRecord.fetchContentTime = Date.now() - functionStartTime; + + const apiResponse = await sendCreateArticleMutation(userId, { + url: finalUrl, + articleSavingRequestId, + preparedDocument: { + document: content, + pageInfo: { + title, + canonicalUrl: finalUrl, + }, + }, + skipParsing: !content, + }); + + logRecord.totalTime = Date.now() - functionStartTime; + logRecord.result = apiResponse.createArticle; + console.log(`parse-page`, logRecord); + + // return res.send({ + // url: finalUrl, + // articleSavingRequestId, + // preparedDocument: { + // document: content, + // pageInfo: { + // title, + // canonicalUrl: finalUrl, + // }, + // }, + // skipParsing: !content, + // timeTaken: Date.now() - functionStartTime, + // }) + } + } catch (e) { + console.log('error', e) + logRecord.error = e.message; + console.log(`Error while retrieving page`, logRecord); + return res.sendStatus(503); + } finally { + if (context) { + await context.close(); + } + } + + return res.sendStatus(200); +} + +function validateUrlString(url) { + const u = new URL(url); + // Make sure the URL is http or https + if (u.protocol !== 'http:' && u.protocol !== 'https:') { + throw new Error('Invalid URL protocol check failed') + } + // Make sure the domain is not localhost + if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') { + throw new Error('Invalid URL is localhost') + } + // Make sure the domain is not a private IP + if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) { + throw new Error('Invalid URL is private ip') + } +} + +function getUrl(req) { + console.log('body', req.body) + const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); + if (!urlStr) { + throw new Error('No URL specified'); + } + + validateUrlString(urlStr); + + const parsed = Url.parse(urlStr); + return parsed.href; +} + +async function retrievePage(url) { + validateUrlString(url); + + const browser = await getBrowserPromise; + logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime }; + + const context = await browser.createIncognitoBrowserContext(); + const page = await context.newPage(); + await page.setUserAgent(userAgentForUrl(url)); + + const client = await page.target().createCDPSession(); + + // intercept request when response headers was received + await client.send('Network.setRequestInterception', { + patterns: [ + { + urlPattern: '*', + resourceType: 'Document', + interceptionStage: 'HeadersReceived', + }, + ], + }); + + const path = require('path'); + const download_path = path.resolve('./download_dir/'); + + await page._client.send('Page.setDownloadBehavior', { + behavior: 'allow', + userDataDir: './', + downloadPath: download_path, + }) + + client.on('Network.requestIntercepted', async e => { + const headers = e.responseHeaders || {}; + + const [contentType] = (headers['content-type'] || headers['Content-Type'] || '') + .toLowerCase() + .split(';'); + const obj = { interceptionId: e.interceptionId }; + + if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) { + // We only check content-type on success responses + // as it doesn't matter what the content type is for things + // like redirects + if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) { + obj['errorReason'] = 'BlockedByClient'; + } + } + + try { + await client.send('Network.continueInterceptedRequest', obj); + // eslint-disable-next-line no-empty + } catch {} + }); + + /* + * Disallow MathJax from running in Puppeteer and modifying the document, + * we shall instead run it in our frontend application to transform any + * mathjax content when present. + */ + await page.setRequestInterception(true); + let requestCount = 0; + // page.on('request', request => { + // if (request.resourceType() === 'font' || request.resourceType() === 'image') { + // request.abort(); + // return; + // } + // if (requestCount++ > 100) { + // request.abort(); + // return; + // } + // if ( + // request.resourceType() === 'script' && + // request.url().toLowerCase().indexOf('mathjax') > -1 + // ) { + // request.abort(); + // } else { + // request.continue(); + // } + // }); + + // Puppeteer fails during download of PDf files, + // so record the failure and use those items + let lastPdfUrl = undefined; + page.on('response', response => { + if (response.headers()['content-type'] === 'application/pdf') { + lastPdfUrl = response.url(); + } + }); + + try { + const response = await page.goto(url, { waitUntil: ['networkidle2'] }); + const finalUrl = response.url(); + const contentType = response.headers()['content-type']; + + logRecord.finalUrl = response.url(); + logRecord.contentType = response.headers()['content-type']; + + return { context, page, response, finalUrl: finalUrl, contentType: contentType }; + } catch (error) { + if (lastPdfUrl) { + return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' }; + } + throw error; + } +} + +async function retrieveHtml(page) { + let domContent = '', title; + try { + title = await page.title(); + logRecord.title = title; + + const pageScrollingStart = Date.now(); + /* scroll with a 5 second timeout */ + await Promise.race([ + new Promise(resolve => { + (async function () { + try { + await page.evaluate(`(async () => { + /* credit: https://github.com/puppeteer/puppeteer/issues/305 */ + return new Promise((resolve, reject) => { + let scrollHeight = document.body.scrollHeight; + let totalHeight = 0; + let distance = 500; + let timer = setInterval(() => { + window.scrollBy(0, distance); + totalHeight += distance; + if(totalHeight >= scrollHeight){ + clearInterval(timer); + resolve(true); + } + }, 10); + }); + })()`); + } catch (e) { + logRecord.scrollError = true; + } finally { + resolve(true); + } + })(); + }), + page.waitForTimeout(1000), //5 second timeout + ]); + logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart }; + + const iframes = {}; + const urls = []; + const framesPromises = []; + const allowedUrls = /instagram\.com/gi; + + for (const frame of page.mainFrame().childFrames()) { + if (frame.url() && allowedUrls.test(frame.url())) { + urls.push(frame.url()); + framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body'))); + } + } + + (await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame)); + + const domContentCapturingStart = Date.now(); + // get document body with all hidden elements removed + domContent = await page.evaluate(iframes => { + const BI_SRC_REGEXP = /url\("(.+?)"\)/gi; + + Array.from(document.body.getElementsByTagName('*')).forEach(el => { + const style = window.getComputedStyle(el); + + // Removing blurred images since they are mostly the copies of lazy loaded ones + if (['img', 'image'].includes(el.tagName.toLowerCase())) { + const filter = style.getPropertyValue('filter'); + if (filter && filter.startsWith('blur')) { + el.parentNode && el.parentNode.removeChild(el); + } + } + + // convert all nodes with background image to img nodes + if (!['', 'none'].includes(style.getPropertyValue('background-image'))) { + const filter = style.getPropertyValue('filter'); + // avoiding image nodes with a blur effect creation + if (filter && filter.startsWith('blur')) { + el && el.parentNode && el.parentNode.removeChild(el); + } else { + const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image')); + // Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage + // More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results + BI_SRC_REGEXP.lastIndex = 0; + + if (matchedSRC && matchedSRC[1] && !el.src) { + // Replacing element only of there are no content inside, b/c might remove important div with content. + // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html + // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image. + if (el.innerHTML.length < 25) { + const img = document.createElement('img'); + img.src = matchedSRC[1]; + el && el.parentNode && el.parentNode.removeChild(el); + } + } + } + } + + if (el.tagName === 'IFRAME') { + if (iframes[el.src]) { + const newNode = document.createElement('div'); + newNode.className = 'omnivore-instagram-embed'; + newNode.innerHTML = iframes[el.src]; + el && el.parentNode && el.parentNode.replaceChild(newNode, el); + } + } + }); + return document.documentElement.innerHTML; + }, iframes); + logRecord.puppeteerSuccess = true; + logRecord.timing = { + ...logRecord.timing, + contenCaptured: Date.now() - domContentCapturingStart, + }; + + // [END puppeteer-block] + } catch (e) { + if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) { + logRecord.blockedByClient = true; + } else { + logRecord.puppeteerSuccess = false; + logRecord.puppeteerError = { + message: e.message, + stack: e.stack, + }; + } + } + return { domContent, title }; +} + +module.exports = fetchContent; diff --git a/packages/content-fetch/image-handler.js b/packages/content-fetch/image-handler.js new file mode 100644 index 000000000..59f132afc --- /dev/null +++ b/packages/content-fetch/image-handler.js @@ -0,0 +1,34 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); + + +exports.imageHandler = { + shouldPrehandle: (url, env) => { + const IMAGE_URL_PATTERN = + /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i + return IMAGE_URL_PATTERN.test(url.toString()) + }, + + prehandle: async (url, env) => { + const title = url.toString().split('/').pop(); + const content = ` + + + ${title} + + + + +
+ ${title} +
+ + ` + + return { title, content }; + } +} diff --git a/packages/content-fetch/medium-handler.js b/packages/content-fetch/medium-handler.js new file mode 100644 index 000000000..704ab0f5b --- /dev/null +++ b/packages/content-fetch/medium-handler.js @@ -0,0 +1,33 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const axios = require('axios'); +const os = require('os'); +const jsdom = require("jsdom"); +const { JSDOM } = jsdom; + +exports.mediumHandler = { + + shouldPrehandle: (url, env) => { + const MEDIUM_URL_MATCH = + /https?:\/\/(www\.)?medium.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/ + const res = MEDIUM_URL_MATCH.test(url.toString()) + return res + }, + + prehandle: async (url, env) => { + console.log('prehandling medium url', url) + + try { + const res = new URL('https://example.org:81/foo'); + myURL.searchParams.delete('source'); + return { url: res } + } catch (error) { + console.error('error prehandling bloomberg url', error) + throw error + } + } +} diff --git a/packages/content-fetch/package.json b/packages/content-fetch/package.json new file mode 100644 index 000000000..ece50efbb --- /dev/null +++ b/packages/content-fetch/package.json @@ -0,0 +1,24 @@ +{ + "name": "@omnivore/content-fetch", + "version": "1.0.0", + "description": "Service that fetches page content from a URL", + "main": "index.js", + "dependencies": { + "@cliqz/adblocker-puppeteer": "^1.23.7", + "ad-block-js": "^0.0.2", + "axios": "^0.26.0", + "dotenv": "^8.2.0", + "express": "^4.17.1", + "jsdom": "^19.0.0", + "jsonwebtoken": "^8.5.1", + "luxon": "^2.3.1", + "puppeteer": "^13.7.0", + "puppeteer-extra": "^3.2.3", + "puppeteer-extra-plugin-adblocker": "^2.12.0", + "puppeteer-extra-plugin-stealth": "^2.9.0" + }, + "scripts": { + "start": "node app.js", + "test": "yarn mocha" + } +} diff --git a/packages/content-fetch/pdf-handler.js b/packages/content-fetch/pdf-handler.js new file mode 100644 index 000000000..1260db287 --- /dev/null +++ b/packages/content-fetch/pdf-handler.js @@ -0,0 +1,21 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const Url = require('url'); + + +exports.pdfHandler = { + + shouldPrehandle: (url, env) => { + const u = Url.parse(url) + const path = u.path.replace(u.search, '') + return path.endsWith('.pdf') + }, + + prehandle: async (url, env) => { + return { contentType: 'application/pdf' }; + } +} diff --git a/packages/content-fetch/t-dot-co-handler.js b/packages/content-fetch/t-dot-co-handler.js new file mode 100644 index 000000000..cbbfb304a --- /dev/null +++ b/packages/content-fetch/t-dot-co-handler.js @@ -0,0 +1,32 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const axios = require('axios'); +const Url = require('url'); + + +exports.tDotCoHandler = { + + shouldResolve: function (url, env) { + const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/; + console.log('should preresolve?', T_DOT_CO_URL_MATCH.test(url), url) + return T_DOT_CO_URL_MATCH.test(url); + }, + + resolve: async function(url, env) { + return await axios.get(url, { maxRedirects: 0, validateStatus: null }) + .then(res => { + return Url.parse(res.headers.location).href; + }).catch((err) => { + console.log('err with t.co url', err); + return undefined; + }); + }, + + shouldPrehandle: (url, env) => { + return false + }, +} diff --git a/packages/content-fetch/test/apple-news-handler.test.js b/packages/content-fetch/test/apple-news-handler.test.js new file mode 100644 index 000000000..4531d720e --- /dev/null +++ b/packages/content-fetch/test/apple-news-handler.test.js @@ -0,0 +1,9 @@ +const { expect } = require('chai') +const { appleNewsHandler } = require('../apple-news-handler') + +describe('open a simple web page', () => { + it('should return a response', async () => { + const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ') + console.log('response', response) + }) +}) diff --git a/packages/content-fetch/test/youtube-handler.test.js b/packages/content-fetch/test/youtube-handler.test.js new file mode 100644 index 000000000..5ba752543 --- /dev/null +++ b/packages/content-fetch/test/youtube-handler.test.js @@ -0,0 +1,12 @@ +const { expect } = require('chai') +const { youtubeHandler } = require('../youtube-handler') + +describe('getVideoId', () => { + it('should parse video id out of a URL', async () => { + expect('BnSUk0je6oo').to.eq(youtubeHandler.getVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s')); + expect('vFD2gu007dc').to.eq(youtubeHandler.getVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1')); + expect('vFD2gu007dc').to.eq(youtubeHandler.getVideoId('https://youtu.be/vFD2gu007dc')); + expect('BMFVCnbRaV4').to.eq(youtubeHandler.getVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share')); + expect('cg9b4RC87LI').to.eq(youtubeHandler.getVideoId('https://youtu.be/cg9b4RC87LI?t=116')); + }) +}) diff --git a/packages/content-fetch/twitter-handler.js b/packages/content-fetch/twitter-handler.js new file mode 100644 index 000000000..99bab6661 --- /dev/null +++ b/packages/content-fetch/twitter-handler.js @@ -0,0 +1,170 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const axios = require('axios'); +const { DateTime } = require('luxon'); + +const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN; +const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/ + +const embeddedTweet = async (url) => { + + const BASE_ENDPOINT = 'https://publish.twitter.com/oembed' + + const apiUrl = new URL(BASE_ENDPOINT) + apiUrl.searchParams.append('url', url); + apiUrl.searchParams.append('omit_script', true); + apiUrl.searchParams.append('dnt', true); + + return await axios.get(apiUrl.toString(), { + headers: { + Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, + redirect: "follow", + }, + }); +}; + +const getTweetFields = () => { + const TWEET_FIELDS = + "&tweet.fields=attachments,author_id,conversation_id,created_at," + + "entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," + + "source,withheld"; + const EXPANSIONS = "&expansions=author_id,attachments.media_keys"; + const USER_FIELDS = + "&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld"; + const MEDIA_FIELDS = + "&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width"; + + return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`; +} + +const getTweetById = async (id) => { + const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/"; + const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields()) + + return await axios.get(apiUrl.toString(), { + headers: { + Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, + redirect: "follow", + }, + }); +}; + +const getUserByUsername = async (username) => { + const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/"; + + const apiUrl = new URL(BASE_ENDPOINT + username) + apiUrl.searchParams.append('user.fields', 'profile_image_url'); + + return await axios.get(apiUrl.toString(), { + headers: { + Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`, + redirect: "follow", + }, + }); +}; + +const titleForTweet = (tweet) => { + return `${tweet.data.author_name} on Twitter` +}; + +const titleForAuthor = (author) => { + return `${author.name} on Twitter` +}; + +const usernameFromStatusUrl = (url) => { + const match = url.toString().match(TWITTER_URL_MATCH) + return match[1] +}; + +const tweetIdFromStatusUrl = (url) => { + const match = url.toString().match(TWITTER_URL_MATCH) + return match[2] +}; + +const formatTimestamp = (timestamp) => { + return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL); +}; + +exports.twitterHandler = { + + shouldPrehandle: (url, env) => { + return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString()) + }, + + // version of the handler that uses the oembed API + // This isn't great as it doesn't work well with our + // readability API. But could potentially give a more consistent + // look to the tweets + // prehandle: async (url, env) => { + // const oeTweet = await embeddedTweet(url) + // const dom = new JSDOM(oeTweet.data.html); + // const bq = dom.window.document.querySelector('blockquote') + // console.log('blockquote:', bq); + + // const title = titleForTweet(oeTweet) + // return { title, content: '
' + bq.innerHTML + '
', url: oeTweet.data.url }; + // } + + prehandle: async (url, env) => { + console.log('prehandling twitter url', url) + + const tweetId = tweetIdFromStatusUrl(url) + const tweetData = (await getTweetById(tweetId)).data; + const authorId = tweetData.data.author_id; + const author = tweetData.includes.users.filter(u => u.id = authorId)[0]; + const title = titleForAuthor(author) + const authorImage = author.profile_image_url.replace('_normal', '_400x400') + + let text = tweetData.data.text; + if (tweetData.data.entities && tweetData.data.entities.urls) { + for (let urlObj of tweetData.data.entities.urls) { + text = text.replace( + urlObj.url, + `${urlObj.display_url}` + ); + } + } + + const front = ` +
+

${text}

+ ` + + var includesHtml = ''; + if (tweetData.includes.media) { + includesHtml = tweetData.includes.media.map(m => { + const linkUrl = m.type == 'photo' ? m.url : url; + const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url; + const mediaOpen = ` + + + + ` + return mediaOpen + }).join('\n'); + } + + const back = ` + — ${author.username} ${author.name} ${formatTimestamp(tweetData.data.created_at)} +
+ ` + const content = ` + + + + + + + + ${front} + ${includesHtml} + ${back} + ` + + return { content, url, title }; + } +} diff --git a/packages/content-fetch/youtube-handler.js b/packages/content-fetch/youtube-handler.js new file mode 100644 index 000000000..1b2286354 --- /dev/null +++ b/packages/content-fetch/youtube-handler.js @@ -0,0 +1,68 @@ +/* eslint-disable no-undef */ +/* eslint-disable no-empty */ +/* eslint-disable @typescript-eslint/explicit-function-return-type */ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable @typescript-eslint/no-require-imports */ +require('dotenv').config(); +const axios = require('axios'); + +const YOUTUBE_URL_MATCH = + /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/ + +exports.youtubeHandler = { + + shouldPrehandle: (url, env) => { + return YOUTUBE_URL_MATCH.test(url.toString()) + }, + + getVideoId: (url) => { + const u = new URL(url); + const videoId = u.searchParams['v'] + if (!videoId) { + const match = url.toString().match(YOUTUBE_URL_MATCH) + if (match === null || match.length < 6 || !match[5]) { + return undefined + } + return match[5] + } + return videoId + }, + + prehandle: async (url, env) => { + const videoId = getVideoId(url) + if (!videoId) { + return {} + } + + const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`) + const oembed = (await axios.get(oembedUrl.toString())).data; + const title = oembed.title; + const ratio = oembed.width / oembed.height; + const thumbnail = oembed.thumbnail_url; + const height = 350; + const width = height * ratio; + + const content = ` + + ${title} + + + + + + + +
+ +
+
+ ${title} + + + ` + + console.log('got video id', videoId) + + return { content, title: 'Youtube Content' }; + } +}