From fc9aa9452cb305a3225296044cfd74fc3c08cf5b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 1 Mar 2022 11:49:38 +0800 Subject: [PATCH] Add a flag in readability to retain table elements in newsletter emails (#152) * add a flag in readability to retain table elements in newsletter emails * remove header of axios newsletters --- packages/api/src/readability.d.ts | 7 +++++++ packages/api/src/services/save_email.ts | 18 +++++++++++------- packages/api/src/utils/axios-handler.ts | 3 ++- packages/api/src/utils/golang-handler.ts | 22 ++++++++++++++++++++++ packages/api/src/utils/parser.ts | 11 ++++++++--- packages/readabilityjs/Readability.js | 3 ++- 6 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 packages/api/src/utils/golang-handler.ts diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index 19ee2e7f4..38be9cc1d 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -131,6 +131,13 @@ declare module '@omnivore/readability' { width?: number, height?: number ) => string + + /** + * By default, Readability will clean all tables from the HTML elements in the + * processed article. But newsletters in emails use tables to display their content. + * By setting this to `true`, these tables will be retained. + */ + keepTables?: boolean } interface ParseResult { diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index e4ef22628..6c4bde97c 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -2,9 +2,9 @@ import { PubsubClient } from '../datalayer/pubsub' import { DataModels } from '../resolvers/types' import { generateSlug, stringToHash, validatedDate } from '../utils/helpers' import { - parsePreparedContent, - parseOriginalContent, parseMetadata, + parseOriginalContent, + parsePreparedContent, } from '../utils/parser' import normalizeUrl from 'normalize-url' import { kx } from '../datalayer/knex_config' @@ -28,12 +28,16 @@ export const saveEmail = async ( input: SaveEmailInput ): Promise => { const url = input.url - const parseResult = await parsePreparedContent(url, { - document: input.originalContent, - pageInfo: { - // can leave this empty for now + const parseResult = await parsePreparedContent( + url, + { + document: input.originalContent, + pageInfo: { + // can leave this empty for now + }, }, - }) + true + ) const title = input.title const content = parseResult.parsedContent?.content || input.originalContent diff --git a/packages/api/src/utils/axios-handler.ts b/packages/api/src/utils/axios-handler.ts index 973379770..dbe4b5619 100644 --- a/packages/api/src/utils/axios-handler.ts +++ b/packages/api/src/utils/axios-handler.ts @@ -17,9 +17,10 @@ export class AxiosHandler { if (k > 0) { el.remove() } else { + // removes the first few rows of the table (the header) // remove the last two rows of the table (they are ads) el.querySelectorAll('tr').forEach((tr, i) => { - if (i >= el.querySelectorAll('tr').length - 2) { + if (i <= 7 || i >= el.querySelectorAll('tr').length - 2) { console.log('removing', tr) tr.remove() } diff --git a/packages/api/src/utils/golang-handler.ts b/packages/api/src/utils/golang-handler.ts new file mode 100644 index 000000000..cc9a6e368 --- /dev/null +++ b/packages/api/src/utils/golang-handler.ts @@ -0,0 +1,22 @@ +import { DOMWindow } from 'jsdom' + +export class GolangHandler { + name = 'golangweekly' + + shouldPrehandle = (url: URL, _dom: DOMWindow): boolean => { + const host = this.name + '.com' + // check if url ends with golangweekly.com + return url.hostname.endsWith(host) + } + + prehandle = (url: URL, dom: DOMWindow): Promise => { + const body = dom.document.querySelector('body') + + // this removes the "Subscribe" button + body?.querySelector('.el-splitbar')?.remove() + // this removes the title + body?.querySelector('.el-masthead')?.remove() + + return Promise.resolve(dom) + } +} diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index ca254517b..ecac81caf 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -12,6 +12,7 @@ import { WikipediaHandler } from './wikipedia-handler' import { SubstackHandler } from './substack-handler' import { AxiosHandler } from './axios-handler' import { BloombergHandler } from './bloomberg-handler' +import { GolangHandler } from './golang-handler' import * as hljs from 'highlightjs' const logger = buildLogger('utils.parse') @@ -48,6 +49,7 @@ const HANDLERS = [ new SubstackHandler(), new AxiosHandler(), new BloombergHandler(), + new GolangHandler(), ] /** Hook that prevents DOMPurify from removing youtube iframes */ @@ -135,7 +137,8 @@ const getPurifiedContent = (html: string): Document => { const getReadabilityResult = ( url: string, html: string, - window: DOMWindow + window: DOMWindow, + isNewsletter?: boolean ): Readability.ParseResult | null => { virtualConsole.removeAllListeners('jsdomError') virtualConsole.on('jsdomError', ({ message, stack: _stack, ...details }) => { @@ -166,6 +169,7 @@ const getReadabilityResult = ( const article = new Readability(document, { debug: DEBUG_MODE, createImageProxyUrl, + keepTables: isNewsletter, }).parse() if (article) { @@ -205,7 +209,8 @@ const applyHandlers = async (url: string, window: DOMWindow): Promise => { export const parsePreparedContent = async ( url: string, - preparedDocument: PreparedDocumentInput + preparedDocument: PreparedDocumentInput, + isNewsletter?: boolean ): Promise => { const logRecord: ArticleParseLogRecord = { url: url, @@ -242,7 +247,7 @@ export const parsePreparedContent = async ( await applyHandlers(url, window) try { - article = getReadabilityResult(url, document, window) + article = getReadabilityResult(url, document, window, isNewsletter) // Format code blocks // TODO: we probably want to move this type of thing diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index fd10623f9..cc1c98db7 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -77,6 +77,7 @@ function Readability(doc, options) { } options = options || {}; this.createImageProxyUrl = options.createImageProxyUrl; + this._keepTables = !!options.keepTables; this._doc = doc; this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__; @@ -852,7 +853,7 @@ Readability.prototype = { // Do these last as the previous stuff may have removed junk // that will affect these - this._cleanConditionally(articleContent, "table"); + !this._keepTables && this._cleanConditionally(articleContent, "table"); this._cleanConditionally(articleContent, "ul"); this._cleanConditionally(articleContent, "div");