Add a flag in readability to retain table elements in newsletter emails (#152)

* add a flag in readability to retain table elements in newsletter emails

* remove header of axios newsletters

packages/api/src/readability.d.ts

@@ -131,6 +131,13 @@ declare module '@omnivore/readability' {
         width?: number,
         height?: number
       ) => string
+
+      /**
+       * By default, Readability will clean all tables from the HTML elements in the
+       * processed article. But newsletters in emails use tables to display their content.
+       * By setting this to `true`, these tables will be retained.
+       */
+      keepTables?: boolean
     }
 
     interface ParseResult {

packages/api/src/services/save_email.ts

@@ -2,9 +2,9 @@ import { PubsubClient } from '../datalayer/pubsub'
 import { DataModels } from '../resolvers/types'
 import { generateSlug, stringToHash, validatedDate } from '../utils/helpers'
 import {
-  parsePreparedContent,
-  parseOriginalContent,
   parseMetadata,
+  parseOriginalContent,
+  parsePreparedContent,
 } from '../utils/parser'
 import normalizeUrl from 'normalize-url'
 import { kx } from '../datalayer/knex_config'
@@ -28,12 +28,16 @@ export const saveEmail = async (
   input: SaveEmailInput
 ): Promise<UserArticleData | undefined> => {
   const url = input.url
-  const parseResult = await parsePreparedContent(url, {
-    document: input.originalContent,
-    pageInfo: {
-      // can leave this empty for now
+  const parseResult = await parsePreparedContent(
+    url,
+    {
+      document: input.originalContent,
+      pageInfo: {
+        // can leave this empty for now
+      },
     },
-  })
+    true
+  )
 
   const title = input.title
   const content = parseResult.parsedContent?.content || input.originalContent

packages/api/src/utils/axios-handler.ts

@@ -17,9 +17,10 @@ export class AxiosHandler {
       if (k > 0) {
         el.remove()
       } else {
+        // removes the first few rows of the table (the header)
         // remove the last two rows of the table (they are ads)
         el.querySelectorAll('tr').forEach((tr, i) => {
-          if (i >= el.querySelectorAll('tr').length - 2) {
+          if (i <= 7 || i >= el.querySelectorAll('tr').length - 2) {
             console.log('removing', tr)
             tr.remove()
           }

packages/api/src/utils/golang-handler.ts

@@ -0,0 +1,22 @@
+import { DOMWindow } from 'jsdom'
+
+export class GolangHandler {
+  name = 'golangweekly'
+
+  shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => {
+    const host = this.name + '.com'
+    // check if url ends with golangweekly.com
+    return url.hostname.endsWith(host)
+  }
+
+  prehandle = (url: URL, dom: DOMWindow): Promise<DOMWindow> => {
+    const body = dom.document.querySelector('body')
+
+    // this removes the "Subscribe" button
+    body?.querySelector('.el-splitbar')?.remove()
+    // this removes the title
+    body?.querySelector('.el-masthead')?.remove()
+
+    return Promise.resolve(dom)
+  }
+}

packages/api/src/utils/parser.ts

@@ -12,6 +12,7 @@ import { WikipediaHandler } from './wikipedia-handler'
 import { SubstackHandler } from './substack-handler'
 import { AxiosHandler } from './axios-handler'
 import { BloombergHandler } from './bloomberg-handler'
+import { GolangHandler } from './golang-handler'
 import * as hljs from 'highlightjs'
 
 const logger = buildLogger('utils.parse')
@@ -48,6 +49,7 @@ const HANDLERS = [
   new SubstackHandler(),
   new AxiosHandler(),
   new BloombergHandler(),
+  new GolangHandler(),
 ]
 
 /** Hook that prevents DOMPurify from removing youtube iframes */
@@ -135,7 +137,8 @@ const getPurifiedContent = (html: string): Document => {
 const getReadabilityResult = (
   url: string,
   html: string,
-  window: DOMWindow
+  window: DOMWindow,
+  isNewsletter?: boolean
 ): Readability.ParseResult | null => {
   virtualConsole.removeAllListeners('jsdomError')
   virtualConsole.on('jsdomError', ({ message, stack: _stack, ...details }) => {
@@ -166,6 +169,7 @@ const getReadabilityResult = (
       const article = new Readability(document, {
         debug: DEBUG_MODE,
         createImageProxyUrl,
+        keepTables: isNewsletter,
       }).parse()
 
       if (article) {
@@ -205,7 +209,8 @@ const applyHandlers = async (url: string, window: DOMWindow): Promise<void> => {
 
 export const parsePreparedContent = async (
   url: string,
-  preparedDocument: PreparedDocumentInput
+  preparedDocument: PreparedDocumentInput,
+  isNewsletter?: boolean
 ): Promise<ParsedContentPuppeteer> => {
   const logRecord: ArticleParseLogRecord = {
     url: url,
@@ -242,7 +247,7 @@ export const parsePreparedContent = async (
   await applyHandlers(url, window)
 
   try {
-    article = getReadabilityResult(url, document, window)
+    article = getReadabilityResult(url, document, window, isNewsletter)
 
     // Format code blocks
     // TODO: we probably want to move this type of thing

packages/readabilityjs/Readability.js

@@ -77,6 +77,7 @@ function Readability(doc, options) {
   }
   options = options || {};
   this.createImageProxyUrl = options.createImageProxyUrl;
+  this._keepTables = !!options.keepTables;
 
   this._doc = doc;
   this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__;
@@ -852,7 +853,7 @@ Readability.prototype = {
 
     // Do these last as the previous stuff may have removed junk
     // that will affect these
-    this._cleanConditionally(articleContent, "table");
+    !this._keepTables && this._cleanConditionally(articleContent, "table");
     this._cleanConditionally(articleContent, "ul");
     this._cleanConditionally(articleContent, "div");