Add a standalone service content fetching

packages/content-fetch/Dockerfile

@@ -0,0 +1,41 @@
+FROM node:14.18-alpine
+
+# Installs latest Chromium (92) package.
+RUN apk add --no-cache \
+      chromium \
+      nss \
+      freetype \
+      harfbuzz \
+      ca-certificates \
+      ttf-freefont \
+      nodejs \
+      yarn
+
+# Tell Puppeteer to skip installing Chrome. We'll be using the installed package.
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+
+# Puppeteer v10.0.0 works with Chromium 92.
+RUN yarn add puppeteer@10.0.0
+
+# Add user so we don't need --no-sandbox.
+RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \
+    && mkdir -p /home/pptruser/Downloads /app \
+    && chown -R pptruser:pptruser /home/pptruser \
+    && chown -R pptruser:pptruser /app
+
+# Run everything after as non-privileged user.
+WORKDIR /app
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+ENV CHROMIUM_PATH /usr/bin/chromium-browser
+ENV LAUNCH_HEADLESS=true
+
+COPY . /app/
+WORKDIR app
+
+RUN yarn install --pure-lockfile
+
+EXPOSE 8080
+
+ENTRYPOINT ["yarn", "start"]

packages/content-fetch/app.js

@@ -0,0 +1,26 @@
+
+
+const express = require('express');
+
+const app = express();
+const fetchContent = require('./fetch-content');
+
+app.use(express.json());
+app.use(express.urlencoded({ extended: true }));
+
+
+app.get('/', (req, res) => {
+  fetchContent(req, res)
+});
+
+app.post('/', (req, res) => {
+  fetchContent(req, res)
+});
+
+const PORT = parseInt(process.env.PORT) || 8080;
+app.listen(PORT, () => {
+  console.log(`App listening on port ${PORT}`);
+  console.log('Press Ctrl+C to quit.');
+});
+
+module.exports = app;

packages/content-fetch/apple-news-handler.js

@@ -0,0 +1,39 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const Url = require('url');
+const axios = require('axios');
+const { promisify } = require('util');
+const { DateTime } = require('luxon');
+const os = require('os');
+const jsdom = require("jsdom");
+const { Cipher } = require('crypto');
+const { JSDOM } = jsdom;
+
+
+
+exports.appleNewsHandler = {
+
+  shouldPrehandle: (url, env) => {
+    const u = new URL(url);
+    if (u.hostname === 'apple.news') {
+      return true;
+    }
+    return false
+  },
+
+  prehandle: async (url, env) => {
+    const MOBILE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36'
+    const response = await axios.get(url, { headers: { 'User-Agent': MOBILE_USER_AGENT }  } );
+    const data = response.data;
+
+    const dom = new JSDOM(data);
+
+    // make sure its a valid URL by wrapping in new URL
+    const u = new URL(dom.window.document.querySelector('span.click-here').parentNode.href);
+    return { url: u.href };
+  }
+}

packages/content-fetch/bloomberg-handler.js

@@ -0,0 +1,40 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const os = require('os');
+const jsdom = require("jsdom");
+const { JSDOM } = jsdom;
+
+exports.bloombergHandler = {
+
+  shouldPrehandle: (url, env) => {
+    const BLOOMBERG_URL_MATCH =
+      /https?:\/\/(www\.)?bloomberg.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
+    return BLOOMBERG_URL_MATCH.test(url.toString())
+  },
+
+  prehandle: async (url, env) => {
+    console.log('prehandling bloomberg url', url)
+
+    try {
+      const response = await axios.get('https://app.scrapingbee.com/api/v1', {
+        params: {
+          'api_key': process.env.SCRAPINGBEE_API_KEY,
+          'url': url,
+          'return_page_source': true,
+          'block_ads': true,
+          'block_resources': false,
+        }
+      })
+      const dom = new JSDOM(response.data);
+      return { title: dom.window.document.title, content: dom.window.document.querySelector('body').innerHTML, url: url }
+    } catch (error) {
+      console.error('error prehandling bloomberg url', error)
+      throw error
+    }
+  }
+}

packages/content-fetch/derstandard-handler.js

@@ -0,0 +1,36 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const jsdom = require("jsdom");
+const { JSDOM } = jsdom;
+
+exports.derstandardHandler = {
+  shouldPrehandle: (url, env) => {
+    const u = new URL(url);
+    return u.hostname === 'www.derstandard.at';
+  },
+
+  prehandle: async (url, env) => {
+    const response = await axios.get(url, {
+      // set cookie to give consent to get the article
+      headers: {
+        'cookie': `DSGVO_ZUSAGE_V1=true; consentUUID=2bacb9c1-1e80-4be0-9f7b-ee987cf4e7b0_6`
+      },
+    });
+    const content = response.data;
+
+    var title = undefined
+    const dom = new JSDOM(content)
+    const titleElement = dom.window.document.querySelector('.article-title')
+    if (!titleElement) {
+      title = titleElement.textContent
+      titleElement.remove()
+    }
+
+    return { content: dom.window.document.body.outerHTML, title: title };
+  }
+}

packages/content-fetch/fetch-content.js

@@ -0,0 +1,585 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const Url = require('url');
+const puppeteer = require('puppeteer-extra');
+const axios = require('axios');
+const jwt = require('jsonwebtoken');
+const { promisify } = require('util');
+const signToken = promisify(jwt.sign);
+const { appleNewsHandler } = require('./apple-news-handler');
+const { twitterHandler } = require('./twitter-handler');
+const { youtubeHandler } = require('./youtube-handler');
+const { tDotCoHandler } = require('./t-dot-co-handler');
+const { pdfHandler } = require('./pdf-handler');
+const { mediumHandler } = require('./medium-handler');
+const { derstandardHandler } = require('./derstandard-handler');
+const { imageHandler } = require('./image-handler');
+
+const MOBILE_USER_AGENT = 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.62 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
+const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
+const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36'
+const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com']
+
+const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf'];
+
+// Add stealth plugin and use defaults (all tricks to hide puppeteer usage)
+// const StealthPlugin = require('puppeteer-extra-plugin-stealth');
+// puppeteer.use(StealthPlugin());
+
+const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker')
+puppeteer.use(AdblockerPlugin({blockTrackers: true}))
+
+
+const userAgentForUrl = (url) => {
+  try {
+    const u = new URL(url);
+    for (const host of NON_BOT_HOSTS) {
+      if (u.hostname.endsWith(host)) {
+        return NON_BOT_DESKTOP_USER_AGENT;
+      }
+    }
+  } catch (e) {
+    console.log('error getting user agent for url', url, e)
+  }
+  return DESKTOP_USER_AGENT
+};
+
+// launch Puppeteer
+const getBrowserPromise = (async () => {
+  return puppeteer.launch({
+    args: ['--no-sandbox'],
+    defaultViewport: { height: 1080, width: 1920 },
+    executablePath: process.env.CHROMIUM_PATH ,
+    headless: true, // process.env.LAUNCH_HEADLESS ? true : false,
+    timeout: 0,
+  });
+})();
+
+let logRecord, functionStartTime;
+
+const uploadToSignedUrl = async ({ id, uploadSignedUrl }, contentType, contentObjUrl) => {
+  const stream = await axios.get(contentObjUrl, { responseType: 'stream' });
+  return await axios.put(uploadSignedUrl, stream.data, {
+    headers: {
+      'Content-Type': contentType,
+    },
+    maxBodyLength: 1000000000,
+    maxContentLength: 100000000,
+  })
+};
+
+const getUploadIdAndSignedUrl = async (userId, url) => {
+  const auth = await signToken({ uid: userId }, process.env.JWT_SECRET);
+  const data = JSON.stringify({
+    query: `mutation UploadFileRequest($input: UploadFileRequestInput!) {
+      uploadFileRequest(input:$input) {
+        ... on UploadFileRequestError {
+          errorCodes
+        }
+        ... on UploadFileRequestSuccess {
+          id
+          uploadSignedUrl
+        }
+      }
+    }`,
+    variables: {
+      input: {
+        url,
+        contentType: 'application/pdf',
+      }
+    }
+  });
+
+  const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data,
+  {
+    headers: {
+      Cookie: `auth=${auth};`,
+      'Content-Type': 'application/json',
+    },
+  });
+  return response.data.data.uploadFileRequest;
+};
+
+const uploadPdf = async (url, userId) => {
+  validateUrlString(url);
+
+  const uploadResult = await getUploadIdAndSignedUrl(userId, url);
+  await uploadToSignedUrl(uploadResult, 'application/pdf', url);
+  return uploadResult.id;
+};
+
+const sendCreateArticleMutation = async (userId, input) => {
+  const data = JSON.stringify({
+    query: `mutation CreateArticle ($input: CreateArticleInput!){
+          createArticle(input:$input){
+            ... on CreateArticleSuccess{
+              createdArticle{
+                id
+            }
+        }
+          ... on CreateArticleError{
+              errorCodes
+          }
+      }
+    }`,
+    variables: {
+      input: Object.assign({}, input , { source: 'puppeteer-parse' }),
+    },
+  });
+
+  const auth = await signToken({ uid: userId }, process.env.JWT_SECRET);
+  const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data,
+  {
+    headers: {
+      Cookie: `auth=${auth};`,
+      'Content-Type': 'application/json',
+    },
+  });
+  console.log('response', response);
+  return response.data.data.createArticle;
+};
+
+const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => {
+  return sendCreateArticleMutation(userId, {
+      url: encodeURI(url),
+      articleSavingRequestId,
+      uploadFileId: uploadFileId,
+    },
+  );
+};
+
+const handlers = {
+  'pdf': pdfHandler,
+  'apple-news': appleNewsHandler,
+  'twitter': twitterHandler,
+  'youtube': youtubeHandler,
+  't-dot-co': tDotCoHandler,
+  'medium': mediumHandler,
+  'derstandard': derstandardHandler,
+  'image': imageHandler,
+};
+
+
+async function fetchContent(req, res) {
+  functionStartTime = Date.now();
+
+  let url = getUrl(req);
+  const userId = (req.query ? req.query.userId : undefined) || (req.body ? req.body.userId : undefined);
+  const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined);
+
+  console.log('user id', userId, 'url', url)
+
+  logRecord = {
+    url,
+    userId,
+    articleSavingRequestId,
+    labels: {
+      source: 'parseContent',
+    },
+  };
+
+  console.log(`Article parsing request`, logRecord);
+
+  if (!url) {
+    logRecord.urlIsInvalid = true;
+    console.log(`Valid URL to parse not specified`, logRecord);
+    return res.sendStatus(400);
+  }
+
+  // if (!userId || !articleSavingRequestId) {
+  //   Object.assign(logRecord, { invalidParams: true, body: req.body, query: req.query });
+  //   console.log(`Invalid parameters`, logRecord);
+  //   return res.sendStatus(400);
+  // }
+
+  // Before we run the regular handlers we check to see if we need tp
+  // pre-resolve the URL. TODO: This should probably happen recursively,
+  // so URLs can be pre-resolved, handled, pre-resolved, handled, etc.
+  for (const [key, handler] of Object.entries(handlers)) {
+    if (handler.shouldResolve && handler.shouldResolve(url)) {
+      try {
+        url = await handler.resolve(url);
+        validateUrlString(url);
+      } catch (err) {
+        console.log('error resolving url with handler', key, err);
+      }
+      break;
+    }
+  }
+
+  // Before we fetch the page we check the handlers, to see if they want
+  // to perform a prefetch action that can modify our requests.
+  // enumerate the handlers and see if any of them want to handle the request
+  const handler = Object.keys(handlers).find(key => {
+    try {
+      return handlers[key].shouldPrehandle(url)
+    } catch (e) {
+      console.log('error with handler: ', key, e);
+    }
+    return false;
+  });
+
+  var title = undefined;
+  var content = undefined;
+  var contentType = undefined;
+
+  if (handler) {
+    try {
+      // The only handler we have now can modify the URL, but in the
+      // future maybe we let it modify content. In that case
+      // we might exit the request early.
+      console.log('pre-handling url with handler: ', handler);
+
+      const result = await handlers[handler].prehandle(url);
+      if (result && result.url) { 
+        url = result.url
+        validateUrlString(url);
+      }
+      if (result && result.title) { title = result.title }
+      if (result && result.content) { content = result.content }
+      if (result && result.contentType) { contentType = result.contentType }
+    } catch (e) {
+      console.log('error with handler: ', handler, e);
+    }
+  }
+
+  var context, page, finalUrl;
+  if ((!content || !title) && contentType !== 'application/pdf') {
+    const result = await retrievePage(url)
+    if (result && result.context) { context = result.context }
+    if (result && result.page) { page = result.page }
+    if (result && result.finalUrl) { finalUrl = result.finalUrl }
+    if (result && result.contentType) { contentType = result.contentType }
+  } else {
+    finalUrl = url
+  }
+
+  try {
+    if (contentType === 'application/pdf') {
+      const uploadedFileId = await uploadPdf(finalUrl, userId);
+      const l = await saveUploadedPdf(userId, finalUrl, uploadedFileId, articleSavingRequestId);
+    } else {
+      if (!content || !title) {
+        const result = await retrieveHtml(page);
+        title = result.title;
+        content = result.domContent;
+      } else {
+        console.log('using prefetched content and title');
+        console.log(content);
+      }
+
+      logRecord.fetchContentTime = Date.now() - functionStartTime;
+
+      const apiResponse = await sendCreateArticleMutation(userId, {
+        url: finalUrl,
+        articleSavingRequestId,
+        preparedDocument: {
+          document: content,
+          pageInfo: {
+            title,
+            canonicalUrl: finalUrl,
+          },
+        },
+        skipParsing: !content,
+      });
+
+      logRecord.totalTime = Date.now() - functionStartTime;
+      logRecord.result = apiResponse.createArticle;
+      console.log(`parse-page`, logRecord);
+
+      // return res.send({
+      //     url: finalUrl,
+      //     articleSavingRequestId,
+      //     preparedDocument: {
+      //       document: content,
+      //       pageInfo: {
+      //         title,
+      //         canonicalUrl: finalUrl,
+      //       },
+      //     },
+      //     skipParsing: !content,
+      //     timeTaken: Date.now() - functionStartTime,
+      //   })
+    }
+  } catch (e) {
+    console.log('error', e)
+    logRecord.error = e.message;
+    console.log(`Error while retrieving page`, logRecord);
+    return res.sendStatus(503);
+  } finally {
+    if (context) {
+      await context.close();
+    }
+  }
+
+  return res.sendStatus(200);
+}
+
+function validateUrlString(url) {
+  const u = new URL(url);
+  // Make sure the URL is http or https
+  if (u.protocol !== 'http:' && u.protocol !== 'https:') {
+    throw new Error('Invalid URL protocol check failed')
+  }
+  // Make sure the domain is not localhost
+  if (u.hostname === 'localhost' || u.hostname === '0.0.0.0') {
+    throw new Error('Invalid URL is localhost')
+  }
+  // Make sure the domain is not a private IP
+  if (/^(10|172\.16|192\.168)\..*/.test(u.hostname)) {
+    throw new Error('Invalid URL is private ip')
+  }
+}
+
+function getUrl(req) {
+  console.log('body', req.body)
+  const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined);
+  if (!urlStr) {
+    throw new Error('No URL specified');
+  }
+
+  validateUrlString(urlStr);
+
+  const parsed = Url.parse(urlStr);
+  return parsed.href;
+}
+
+async function retrievePage(url) {
+  validateUrlString(url);
+
+  const browser = await getBrowserPromise;
+  logRecord.timing = { ...logRecord.timing, browserOpened: Date.now() - functionStartTime };
+
+  const context = await browser.createIncognitoBrowserContext();
+  const page = await context.newPage();
+  await page.setUserAgent(userAgentForUrl(url));
+
+  const client = await page.target().createCDPSession();
+
+  // intercept request when response headers was received
+  await client.send('Network.setRequestInterception', {
+    patterns: [
+      {
+        urlPattern: '*',
+        resourceType: 'Document',
+        interceptionStage: 'HeadersReceived',
+      },
+    ],
+  });
+
+  const path = require('path');
+  const download_path = path.resolve('./download_dir/');
+
+  await page._client.send('Page.setDownloadBehavior', {
+      behavior: 'allow',
+      userDataDir: './',
+      downloadPath: download_path,
+  })
+
+  client.on('Network.requestIntercepted', async e => {
+    const headers = e.responseHeaders || {};
+
+    const [contentType] = (headers['content-type'] || headers['Content-Type'] || '')
+      .toLowerCase()
+      .split(';');
+    const obj = { interceptionId: e.interceptionId };
+
+    if (e.responseStatusCode >= 200 && e.responseStatusCode < 300) {
+      // We only check content-type on success responses
+      // as it doesn't matter what the content type is for things
+      // like redirects
+      if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
+        obj['errorReason'] = 'BlockedByClient';
+      }
+    }
+
+    try {
+      await client.send('Network.continueInterceptedRequest', obj);
+      // eslint-disable-next-line no-empty
+    } catch {}
+  });
+
+  /*
+    * Disallow MathJax from running in Puppeteer and modifying the document,
+    * we shall instead run it in our frontend application to transform any
+    * mathjax content when present.
+    */
+  await page.setRequestInterception(true);
+  let requestCount = 0;
+  // page.on('request', request => {
+  //   if (request.resourceType() === 'font' || request.resourceType() === 'image') {
+  //     request.abort();
+  //     return;
+  //   }
+  //   if (requestCount++ > 100) {
+  //     request.abort();
+  //     return;
+  //   }
+  //   if (
+  //     request.resourceType() === 'script' &&
+  //     request.url().toLowerCase().indexOf('mathjax') > -1
+  //   ) {
+  //     request.abort();
+  //   } else {
+  //     request.continue();
+  //   }
+  // });
+
+  // Puppeteer fails during download of PDf files,
+  // so record the failure and use those items
+  let lastPdfUrl = undefined;
+  page.on('response', response => {
+    if (response.headers()['content-type'] === 'application/pdf') {
+      lastPdfUrl = response.url();
+    }
+  });
+
+  try {
+    const response = await page.goto(url, { waitUntil: ['networkidle2'] });
+    const finalUrl = response.url();
+    const contentType = response.headers()['content-type'];
+
+    logRecord.finalUrl = response.url();
+    logRecord.contentType = response.headers()['content-type'];
+
+    return { context, page, response, finalUrl: finalUrl, contentType: contentType };
+  } catch (error) {
+    if (lastPdfUrl) {
+      return { context, page, finalUrl: lastPdfUrl, contentType: 'application/pdf' };
+    }
+    throw error;
+  }
+}
+
+async function retrieveHtml(page) {
+  let domContent = '', title;
+  try {
+    title = await page.title();
+    logRecord.title = title;
+
+    const pageScrollingStart = Date.now();
+    /* scroll with a 5 second timeout */
+    await Promise.race([
+      new Promise(resolve => {
+        (async function () {
+          try {
+            await page.evaluate(`(async () => {
+                /* credit: https://github.com/puppeteer/puppeteer/issues/305 */
+                return new Promise((resolve, reject) => {
+                  let scrollHeight = document.body.scrollHeight;
+                  let totalHeight = 0;
+                  let distance = 500;
+                  let timer = setInterval(() => {
+                    window.scrollBy(0, distance);
+                    totalHeight += distance;
+                    if(totalHeight >= scrollHeight){
+                      clearInterval(timer);
+                      resolve(true);
+                    }
+                  }, 10);
+                });
+              })()`);
+          } catch (e) {
+            logRecord.scrollError = true;
+          } finally {
+            resolve(true);
+          }
+        })();
+      }),
+      page.waitForTimeout(1000), //5 second timeout
+    ]);
+    logRecord.timing = { ...logRecord.timing, pageScrolled: Date.now() - pageScrollingStart };
+
+    const iframes = {};
+    const urls = [];
+    const framesPromises = [];
+    const allowedUrls = /instagram\.com/gi;
+
+    for (const frame of page.mainFrame().childFrames()) {
+      if (frame.url() && allowedUrls.test(frame.url())) {
+        urls.push(frame.url());
+        framesPromises.push(frame.evaluate(el => el.innerHTML, await frame.$('body')));
+      }
+    }
+
+    (await Promise.all(framesPromises)).forEach((frame, index) => (iframes[urls[index]] = frame));
+
+    const domContentCapturingStart = Date.now();
+    // get document body with all hidden elements removed
+    domContent = await page.evaluate(iframes => {
+      const BI_SRC_REGEXP = /url\("(.+?)"\)/gi;
+
+      Array.from(document.body.getElementsByTagName('*')).forEach(el => {
+        const style = window.getComputedStyle(el);
+
+        // Removing blurred images since they are mostly the copies of lazy loaded ones
+        if (['img', 'image'].includes(el.tagName.toLowerCase())) {
+          const filter = style.getPropertyValue('filter');
+          if (filter && filter.startsWith('blur')) {
+            el.parentNode && el.parentNode.removeChild(el);
+          }
+        }
+
+        // convert all nodes with background image to img nodes
+        if (!['', 'none'].includes(style.getPropertyValue('background-image'))) {
+          const filter = style.getPropertyValue('filter');
+          // avoiding image nodes with a blur effect creation
+          if (filter && filter.startsWith('blur')) {
+            el && el.parentNode && el.parentNode.removeChild(el);
+          } else {
+            const matchedSRC = BI_SRC_REGEXP.exec(style.getPropertyValue('background-image'));
+            // Using "g" flag with a regex we have to manually break down lastIndex to zero after every usage
+            // More details here: https://stackoverflow.com/questions/1520800/why-does-a-regexp-with-global-flag-give-wrong-results
+            BI_SRC_REGEXP.lastIndex = 0;
+
+            if (matchedSRC && matchedSRC[1] && !el.src) {
+              // Replacing element only of there are no content inside, b/c might remove important div with content.
+              // Article example: http://www.josiahzayner.com/2017/01/genetic-designer-part-i.html
+              // DIV with class "content-inner" has `url("https://resources.blogblog.com/blogblog/data/1kt/travel/bg_container.png")` background image.
+              if (el.innerHTML.length < 25) {
+                const img = document.createElement('img');
+                img.src = matchedSRC[1];
+                el && el.parentNode && el.parentNode.removeChild(el);
+              }
+            }
+          }
+        }
+
+        if (el.tagName === 'IFRAME') {
+          if (iframes[el.src]) {
+            const newNode = document.createElement('div');
+            newNode.className = 'omnivore-instagram-embed';
+            newNode.innerHTML = iframes[el.src];
+            el && el.parentNode && el.parentNode.replaceChild(newNode, el);
+          }
+        }
+      });
+      return document.documentElement.innerHTML;
+    }, iframes);
+    logRecord.puppeteerSuccess = true;
+    logRecord.timing = {
+      ...logRecord.timing,
+      contenCaptured: Date.now() - domContentCapturingStart,
+    };
+
+    // [END puppeteer-block]
+  } catch (e) {
+    if (e.message.startsWith('net::ERR_BLOCKED_BY_CLIENT at ')) {
+      logRecord.blockedByClient = true;
+    } else {
+      logRecord.puppeteerSuccess = false;
+      logRecord.puppeteerError = {
+        message: e.message,
+        stack: e.stack,
+      };
+    }
+  }
+  return { domContent, title };
+}
+
+module.exports = fetchContent;

packages/content-fetch/image-handler.js

@@ -0,0 +1,34 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+
+
+exports.imageHandler = {
+  shouldPrehandle: (url, env) => {
+    const IMAGE_URL_PATTERN =
+      /(https?:\/\/.*\.(?:jpg|jpeg|png|webp))/i
+    return IMAGE_URL_PATTERN.test(url.toString())
+  },
+
+  prehandle: async (url, env) => {
+    const title = url.toString().split('/').pop();
+    const content = `
+      <html>
+        <head>
+          <title>${title}</title>
+          <meta property="og:image" content="${url}" />
+          <meta property="og:title" content="${title}" />
+        </head>
+        <body>
+          <div>
+            <img src="${url}" alt="${title}">
+          </div>
+        </body>
+      </html>`
+
+    return { title, content };
+  }
+}

packages/content-fetch/medium-handler.js

@@ -0,0 +1,33 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const os = require('os');
+const jsdom = require("jsdom");
+const { JSDOM } = jsdom;
+
+exports.mediumHandler = {
+
+  shouldPrehandle: (url, env) => {
+    const MEDIUM_URL_MATCH =
+      /https?:\/\/(www\.)?medium.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/
+    const res =  MEDIUM_URL_MATCH.test(url.toString())
+    return res
+  },
+
+  prehandle: async (url, env) => {
+    console.log('prehandling medium url', url)
+
+    try {
+      const res = new URL('https://example.org:81/foo');
+      myURL.searchParams.delete('source');
+      return { url: res }
+    } catch (error) {
+      console.error('error prehandling bloomberg url', error)
+      throw error
+    }
+  }
+}

packages/content-fetch/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@omnivore/content-fetch",
+  "version": "1.0.0",
+  "description": "Service that fetches page content from a URL",
+  "main": "index.js",
+  "dependencies": {
+    "@cliqz/adblocker-puppeteer": "^1.23.7",
+    "ad-block-js": "^0.0.2",
+    "axios": "^0.26.0",
+    "dotenv": "^8.2.0",
+    "express": "^4.17.1",
+    "jsdom": "^19.0.0",
+    "jsonwebtoken": "^8.5.1",
+    "luxon": "^2.3.1",
+    "puppeteer": "^13.7.0",
+    "puppeteer-extra": "^3.2.3",
+    "puppeteer-extra-plugin-adblocker": "^2.12.0",
+    "puppeteer-extra-plugin-stealth": "^2.9.0"
+  },
+  "scripts": {
+    "start": "node app.js",
+    "test": "yarn mocha"
+  }
+}

packages/content-fetch/pdf-handler.js

@@ -0,0 +1,21 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const Url = require('url');
+
+
+exports.pdfHandler = {
+
+  shouldPrehandle: (url, env) => {
+    const u = Url.parse(url)
+    const path = u.path.replace(u.search, '')
+    return path.endsWith('.pdf')
+  },
+
+  prehandle: async (url, env) => {
+    return { contentType: 'application/pdf' };
+  }
+}

packages/content-fetch/t-dot-co-handler.js

@@ -0,0 +1,32 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const Url = require('url');
+
+
+exports.tDotCoHandler = {
+
+  shouldResolve: function (url, env) {
+    const T_DOT_CO_URL_MATCH = /^https:\/\/(?:www\.)?t\.co\/.*$/;
+    console.log('should preresolve?', T_DOT_CO_URL_MATCH.test(url), url)
+    return T_DOT_CO_URL_MATCH.test(url);
+  },
+
+  resolve: async function(url, env) {
+    return await axios.get(url, { maxRedirects: 0, validateStatus: null })
+      .then(res => {
+        return Url.parse(res.headers.location).href;
+      }).catch((err) => {
+        console.log('err with t.co url', err);
+        return undefined;
+      });
+  },
+
+  shouldPrehandle: (url, env) => {
+    return false
+  },
+}

packages/content-fetch/test/apple-news-handler.test.js

@@ -0,0 +1,9 @@
+const { expect } = require('chai')
+const { appleNewsHandler } = require('../apple-news-handler')
+
+describe('open a simple web page', () => {
+  it('should return a response', async () => {
+    const response = await appleNewsHandler.prehandle('https://apple.news/AxjzaZaPvSn23b67LhXI5EQ')
+    console.log('response', response)
+  })
+})

packages/content-fetch/test/youtube-handler.test.js

@@ -0,0 +1,12 @@
+const { expect } = require('chai')
+const { youtubeHandler } = require('../youtube-handler')
+
+describe('getVideoId', () => {
+  it('should parse video id out of a URL', async () => {
+    expect('BnSUk0je6oo').to.eq(youtubeHandler.getVideoId('https://www.youtube.com/watch?v=BnSUk0je6oo&t=269s'));
+    expect('vFD2gu007dc').to.eq(youtubeHandler.getVideoId('https://www.youtube.com/watch?v=vFD2gu007dc&list=RDvFD2gu007dc&start_radio=1'));
+    expect('vFD2gu007dc').to.eq(youtubeHandler.getVideoId('https://youtu.be/vFD2gu007dc'));
+    expect('BMFVCnbRaV4').to.eq(youtubeHandler.getVideoId('https://youtube.com/watch?v=BMFVCnbRaV4&feature=share'));
+    expect('cg9b4RC87LI').to.eq(youtubeHandler.getVideoId('https://youtu.be/cg9b4RC87LI?t=116'));
+  })
+})

packages/content-fetch/twitter-handler.js

@@ -0,0 +1,170 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+const { DateTime } = require('luxon');
+
+const TWITTER_BEARER_TOKEN = process.env.TWITTER_BEARER_TOKEN;
+const TWITTER_URL_MATCH = /twitter\.com\/(?:#!\/)?(\w+)\/status(?:es)?\/(\d+)(?:\/.*)?/
+
+const embeddedTweet = async (url) => {
+
+  const BASE_ENDPOINT = 'https://publish.twitter.com/oembed'
+
+  const apiUrl = new URL(BASE_ENDPOINT)
+  apiUrl.searchParams.append('url', url);
+  apiUrl.searchParams.append('omit_script', true);
+  apiUrl.searchParams.append('dnt', true);
+
+  return await axios.get(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: "follow",
+    },
+  });
+};
+
+const getTweetFields = () => {
+  const TWEET_FIELDS =
+  "&tweet.fields=attachments,author_id,conversation_id,created_at," +
+  "entities,geo,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets," +
+  "source,withheld";
+  const EXPANSIONS = "&expansions=author_id,attachments.media_keys";
+  const USER_FIELDS =
+  "&user.fields=created_at,description,entities,location,pinned_tweet_id,profile_image_url,protected,public_metrics,url,verified,withheld";
+  const MEDIA_FIELDS =
+  "&media.fields=duration_ms,height,preview_image_url,url,media_key,public_metrics,width";
+
+  return `${TWEET_FIELDS}${EXPANSIONS}${USER_FIELDS}${MEDIA_FIELDS}`;
+}
+
+const getTweetById = async (id) => {
+  const BASE_ENDPOINT = "https://api.twitter.com/2/tweets/";
+  const apiUrl = new URL(BASE_ENDPOINT + id + '?' + getTweetFields())
+
+  return await axios.get(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: "follow",
+    },
+  });
+};
+
+const getUserByUsername = async (username) => {
+  const BASE_ENDPOINT = "https://api.twitter.com/2/users/by/username/";
+
+  const apiUrl = new URL(BASE_ENDPOINT + username)
+  apiUrl.searchParams.append('user.fields', 'profile_image_url');
+
+  return await axios.get(apiUrl.toString(), {
+    headers: {
+      Authorization: `Bearer ${TWITTER_BEARER_TOKEN}`,
+      redirect: "follow",
+    },
+  });
+};
+
+const titleForTweet = (tweet) => {
+  return `${tweet.data.author_name} on Twitter`
+};
+
+const titleForAuthor = (author) => {
+  return `${author.name} on Twitter`
+};
+
+const usernameFromStatusUrl = (url) => {
+  const match = url.toString().match(TWITTER_URL_MATCH)
+  return match[1]
+};
+
+const tweetIdFromStatusUrl = (url) => {
+  const match = url.toString().match(TWITTER_URL_MATCH)
+  return match[2]
+};
+
+const formatTimestamp = (timestamp) => {
+  return DateTime.fromJSDate(new Date(timestamp)).toLocaleString(DateTime.DATETIME_FULL);
+};
+
+exports.twitterHandler = {
+
+  shouldPrehandle: (url, env) => {
+    return TWITTER_BEARER_TOKEN && TWITTER_URL_MATCH.test(url.toString())
+  },
+
+  // version of the handler that uses the oembed API
+  // This isn't great as it doesn't work well with our
+  // readability API. But could potentially give a more consistent
+  // look to the tweets
+  // prehandle: async (url, env) => {
+  //   const oeTweet = await embeddedTweet(url)
+  //   const dom = new JSDOM(oeTweet.data.html);
+  //   const bq = dom.window.document.querySelector('blockquote')
+  //   console.log('blockquote:', bq);
+
+  //   const title = titleForTweet(oeTweet)
+  //   return { title, content: '<div>' + bq.innerHTML + '</div>', url: oeTweet.data.url };
+  // }
+
+  prehandle: async (url, env) => {
+    console.log('prehandling twitter url', url)
+
+    const tweetId = tweetIdFromStatusUrl(url)
+    const tweetData = (await getTweetById(tweetId)).data;
+    const authorId = tweetData.data.author_id;
+    const author = tweetData.includes.users.filter(u => u.id = authorId)[0];
+    const title = titleForAuthor(author)
+    const authorImage = author.profile_image_url.replace('_normal', '_400x400')
+
+    let text = tweetData.data.text;
+    if (tweetData.data.entities && tweetData.data.entities.urls) {
+      for (let urlObj of tweetData.data.entities.urls) {
+        text = text.replace(
+          urlObj.url,
+          `<a href="${urlObj.expanded_url}">${urlObj.display_url}</a>`
+        );
+      }
+    }
+
+    const front = `
+    <div>
+      <p>${text}</p>
+    `
+
+    var includesHtml =  '';
+    if (tweetData.includes.media) {
+      includesHtml = tweetData.includes.media.map(m => {
+        const linkUrl = m.type == 'photo' ? m.url : url;
+        const previewUrl = m.type == 'photo' ? m.url : m.preview_image_url;
+        const mediaOpen = `<a class="media-link" href=${linkUrl}>
+          <picture>
+            <img class="tweet-img" src=${previewUrl} />
+          </picture>
+          </a>`
+        return mediaOpen
+      }).join('\n');
+   }
+
+   const back = `
+       — <a href="https://twitter.com/${author.username}">${author.username}</a> ${author.name} <a href="${url}">${formatTimestamp(tweetData.data.created_at)}</a>
+    </div>
+    `
+    const content = `
+    <head>
+      <meta property="og:image" content="${authorImage}" />
+      <meta property="og:image:secure_url" content="${authorImage}" />
+      <meta property="og:title" content="${title}" />
+      <meta property="og:description" content="${tweetData.data.text}" />
+    </head>
+    <body>
+      ${front}
+      ${includesHtml}
+      ${back}
+    </body>`
+
+    return { content, url, title };
+  }
+}

packages/content-fetch/youtube-handler.js

@@ -0,0 +1,68 @@
+/* eslint-disable no-undef */
+/* eslint-disable no-empty */
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable @typescript-eslint/no-require-imports */
+require('dotenv').config();
+const axios = require('axios');
+
+const YOUTUBE_URL_MATCH =
+  /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/
+
+exports.youtubeHandler = {
+
+  shouldPrehandle: (url, env) => {
+    return YOUTUBE_URL_MATCH.test(url.toString())
+  },
+
+  getVideoId: (url) => {
+    const u = new URL(url);
+    const videoId = u.searchParams['v']
+    if (!videoId) {
+      const match = url.toString().match(YOUTUBE_URL_MATCH)
+      if (match === null || match.length < 6 || !match[5]) {
+        return undefined
+      }
+      return match[5]
+    }
+    return videoId
+  },
+
+  prehandle: async (url, env) => {
+    const videoId = getVideoId(url)
+    if (!videoId) {
+      return {}
+    }
+
+    const oembedUrl = `https://www.youtube.com/oembed?format=json&url=` + encodeURIComponent(`https://www.youtube.com/watch?v=${videoId}`)
+    const oembed = (await axios.get(oembedUrl.toString())).data;
+    const title = oembed.title;
+    const ratio = oembed.width / oembed.height;
+    const thumbnail = oembed.thumbnail_url;
+    const height = 350;
+    const width = height * ratio;
+
+    const content = `
+    <html>
+      <head><title>${title}</title>
+      <meta property="og:image" content="${thumbnail}" />
+      <meta property="og:image:secure_url" content="${thumbnail}" />
+      <meta property="og:title" content="${title}" />
+      <meta property="og:description" content="" />
+      <meta property="og:article:author" content="${oembed.author_name}" />
+      </head>
+      <body>
+      <center>
+      <iframe width="${width}" height="${height}" src="https://www.youtube.com/embed/${videoId}" title="${title}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
+      </center>
+      <br />
+        <a href="${url}">${title}</a>
+        <div itemscope="" itemprop="author" itemtype="http://schema.org/Person">By <a href="${oembed.author_url}">${oembed.author_name}</a></div>
+      </body>
+    </html>`
+
+    console.log('got video id', videoId)
+
+    return { content, title: 'Youtube Content' };
+  }
+}