Added option for Firefox for parser. Was having issues with Chromium on Docker.

packages/content-fetch/Dockerfile

@@ -4,6 +4,7 @@ LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
 # Installs latest Chromium package.
 RUN apt-get update && apt-get install -y \
     chromium \
+    firefox-esr \
     ca-certificates \
     nodejs \
     yarn \
@@ -14,6 +15,7 @@ RUN apt-get update && apt-get install -y \
 WORKDIR /app
 
 ENV CHROMIUM_PATH /usr/bin/chromium
+ENV FIREFOX_PATH /usr/bin/firefox
 ENV LAUNCH_HEADLESS=true
 
 COPY package.json .
@@ -45,5 +47,13 @@ RUN yarn install --pure-lockfile --production
 
 EXPOSE 8080
 
-CMD ["yarn", "workspace", "@omnivore/content-fetch", "start"]
+# In Firefox we can't use the adblocking sites. Adding them to the hosts file of the docker seems to work.
+RUN wget https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+RUN echo "#!/bin/bash \n\
+cat hosts >> /etc/hosts \n\
+yarn workspace @omnivore/content-fetch start" >> ./start.sh
+
+RUN chmod +x ./start.sh
+
+CMD ["./start.sh"]
 

packages/puppeteer-parse/src/browser.ts

@@ -3,8 +3,10 @@ import puppeteer from 'puppeteer-extra'
 import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker'
 import StealthPlugin from 'puppeteer-extra-plugin-stealth'
 
-puppeteer.use(StealthPlugin())
-puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
+if (process.env['USE_FIREFOX'] != 'true') {
+  puppeteer.use(StealthPlugin())
+  puppeteer.use(AdblockerPlugin({ blockTrackers: true }))
+}
 
 let browserInstance: Browser | null = null
 
@@ -51,15 +53,22 @@ export const getBrowser = async (): Promise<Browser> => {
       isMobile: false,
       width: 1920,
     },
-    executablePath: process.env.CHROMIUM_PATH,
+    ignoreHTTPSErrors: true,
+    executablePath:
+      process.env.USE_FIREFOX == 'true'
+        ? process.env.FIREFOX_PATH
+        : process.env.CHROMIUM_PATH,
     // run in shell mode if headless
-    headless: process.env.LAUNCH_HEADLESS === 'true' ? 'shell' : false,
-    timeout: 10_000, // 10 seconds
-    dumpio: true, // show console logs in the terminal
+    headless: true,
+    browser: process.env['USE_FIREFOX'] == 'true' ? 'firefox' : 'chrome',
+    product: process.env['USE_FIREFOX'] == 'true' ? 'firefox' : 'chrome',
+    timeout: 30000,
+    dumpio: true,
+
     // filter out targets
     targetFilter: (target: Target) =>
       target.type() !== 'other' || !!target.url(),
-  })) as Browser
+  })) as unknown as Browser
 
   const version = await browserInstance.version()
   console.log('Browser started', version)

packages/puppeteer-parse/src/index.ts

@@ -144,7 +144,7 @@ function getUrl(urlStr: string) {
   return parsed.href
 }
 
-const waitForDOMToSettle = (page: Page, timeoutMs = 2500, debounceMs = 1000) =>
+const waitForDOMToSettle = (page: Page, timeoutMs = 5000, debounceMs = 1000) =>
   page.evaluate(
     (timeoutMs, debounceMs) => {
       const debounce = (func: (...args: unknown[]) => void, ms = 1000) => {
@@ -223,83 +223,89 @@ async function retrievePage(
     }
 
     // set timezone for the page
-    if (timezone) {
-      await page.emulateTimezone(timezone)
-    }
-
-    const client = await page.createCDPSession()
-
-    const downloadPath = path.resolve('./download_dir/')
-    await client.send('Page.setDownloadBehavior', {
-      behavior: 'allow',
-      downloadPath,
-    })
-
-    // intercept request when response headers was received
-    await client.send('Network.setRequestInterception', {
-      patterns: [
-        {
-          urlPattern: '*',
-          resourceType: 'Document',
-          interceptionStage: 'HeadersReceived',
-        },
-      ],
-    })
-
-    client.on(
-      'Network.requestIntercepted',
-      (e: Protocol.Network.RequestInterceptedEvent) => {
-        ;(async () => {
-          const headers = e.responseHeaders || {}
-
-          const [contentType] = (
-            headers['content-type'] ||
-            headers['Content-Type'] ||
-            ''
-          )
-            .toLowerCase()
-            .split(';')
-          const obj: Protocol.Network.ContinueInterceptedRequestRequest = {
-            interceptionId: e.interceptionId,
-          }
-
-          if (
-            e.responseStatusCode &&
-            e.responseStatusCode >= 200 &&
-            e.responseStatusCode < 300
-          ) {
-            // We only check content-type on success responses
-            // as it doesn't matter what the content type is for things
-            // like redirects
-            if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
-              obj['errorReason'] = 'BlockedByClient'
-            }
-          }
-
-          try {
-            await client.send('Network.continueInterceptedRequest', obj)
-          } catch {
-            // ignore
-          }
-        })()
+    if (process.env['USE_FIREFOX'] !== 'true') {
+      if (timezone) {
+        await page.emulateTimezone(timezone)
       }
-    )
+
+      const client = await page.createCDPSession()
+
+      const downloadPath = path.resolve('./download_dir/')
+      await client.send('Page.setDownloadBehavior', {
+        behavior: 'allow',
+        downloadPath,
+      })
+
+      // intercept request when response headers was received
+      await client.send('Network.setRequestInterception', {
+        patterns: [
+          {
+            urlPattern: '*',
+            resourceType: 'Document',
+            interceptionStage: 'HeadersReceived',
+          },
+        ],
+      })
+
+      client.on(
+        'Network.requestIntercepted',
+        (e: Protocol.Network.RequestInterceptedEvent) => {
+          ;(async () => {
+            const headers = e.responseHeaders || {}
+
+            const [contentType] = (
+              headers['content-type'] ||
+              headers['Content-Type'] ||
+              ''
+            )
+              .toLowerCase()
+              .split(';')
+            const obj: Protocol.Network.ContinueInterceptedRequestRequest = {
+              interceptionId: e.interceptionId,
+            }
+
+            if (
+              e.responseStatusCode &&
+              e.responseStatusCode >= 200 &&
+              e.responseStatusCode < 300
+            ) {
+              // We only check content-type on success responses
+              // as it doesn't matter what the content type is for things
+              // like redirects
+              if (contentType && !ALLOWED_CONTENT_TYPES.includes(contentType)) {
+                obj['errorReason'] = 'BlockedByClient'
+              }
+            }
+
+            try {
+              await client.send('Network.continueInterceptedRequest', obj)
+            } catch {
+              // ignore
+            }
+          })()
+        }
+      )
+    }
 
     /*
      * Disallow MathJax from running in Puppeteer and modifying the document,
      * we shall instead run it in our frontend application to transform any
      * mathjax content when present.
      */
-    await page.setRequestInterception(true)
+
     let requestCount = 0
     const failedRequests = new Set()
+    page.removeAllListeners('request')
     page.on('request', (request) => {
       ;(async () => {
-        if (request.resourceType() === 'font') {
+        if (request.isInterceptResolutionHandled()) return
+        // since .requestType() is not FF compatible, look for font files.
+        if (request.url().toLowerCase().includes('.woff2')) {
           // Disallow fonts from loading
           return request.abort()
         }
-        if (requestCount++ > 50) {
+
+        if (requestCount++ > 100) {
           return request.abort()
         }
 
@@ -308,7 +314,6 @@ async function retrievePage(
         }
 
         if (
-          request.resourceType() === 'script' &&
           request.url().toLowerCase().indexOf('mathjax') > -1
         ) {
           return request.abort()
@@ -317,6 +322,8 @@ async function retrievePage(
         await request.continue()
       })()
     })
+    await page.setRequestInterception(true)
+
 
     page.on('response', (response) => {
       if (!response.ok()) {
@@ -338,6 +345,8 @@ async function retrievePage(
 
     console.log('Waited for content to load, waiting for DOM to settle.')
     await waitForDOMToSettle(page)
+    // Just wait for a few seconds to allow the dom to resolve.
+    // await new Promise((r) => setTimeout(r, 2500))
 
     if (!response) {
       throw new Error('No response from page')

self-hosting/docker-compose/docker-compose.yml

@@ -89,12 +89,15 @@ services:
       - .env
 
   content-fetch:
+    platform: linux/amd64
     build:
       context: ../../
       dockerfile: ./packages/content-fetch/Dockerfile
     container_name: "omnivore-content-fetch"
     ports:
       - "9090:8080"
+    environment:
+      - USE_FIREFOX=true # Using Firefox here because the official chrome version seems to freeze a lot in Docker.
     env_file:
       - .env
     depends_on: