From 6a49689d1eb08f2a7f5c3ff0a21cedd6125e6726 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 27 Dec 2022 21:22:49 +0800 Subject: [PATCH 1/7] Add readability in puppeteer-parse --- packages/puppeteer-parse/index.js | 17 +++++++++++++++-- packages/puppeteer-parse/package.json | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index efce2c6e0..ae2dd942b 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -13,6 +13,7 @@ const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); const { preHandleContent } = require("@omnivore/content-handler"); +const { Readability } = require("@omnivore/readability"); const puppeteer = require('puppeteer-extra'); @@ -280,6 +281,8 @@ async function fetchContent(req, res) { logRecord.fetchContentTime = Date.now() - functionStartTime; + const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; + const apiResponse = await sendCreateArticleMutation(userId, { url: finalUrl, articleSavingRequestId, @@ -290,7 +293,8 @@ async function fetchContent(req, res) { canonicalUrl: finalUrl, }, }, - skipParsing: !content, + skipParsing: !!readabilityResult, + readabilityResult, }); logRecord.totalTime = Date.now() - functionStartTime; @@ -306,6 +310,8 @@ async function fetchContent(req, res) { const content = sbResult.domContent; logRecord.fetchContentTime = Date.now() - functionStartTime; + const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; + const apiResponse = await sendCreateArticleMutation(userId, { url: sbUrl, articleSavingRequestId, @@ -316,7 +322,8 @@ async function fetchContent(req, res) { canonicalUrl: sbUrl, }, }, - skipParsing: !content, + skipParsing: !!readabilityResult, + readabilityResult, }); logRecord.totalTime = Date.now() - functionStartTime; @@ -758,6 +765,12 @@ async function preview(req, res) { return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); } +async function getReadabilityResult(url, domContent) { + const document = parseHTML(domContent).document; + const readability = new Readability(document, { url }); + return readability.parse(); +} + module.exports = { fetchContent, preview, diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 42b3a10a4..8798377a9 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -6,6 +6,7 @@ "dependencies": { "@google-cloud/storage": "^5.18.1", "@omnivore/content-handler": "1.0.0", + "@omnivore/readability": "1.0.0", "axios": "^0.27.2", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", From e9b31e375f3987c1f50b8aae60ccd8597271437e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 27 Dec 2022 21:59:31 +0800 Subject: [PATCH 2/7] Update savePage API to accept parsed content in the param --- packages/api/src/generated/graphql.ts | 18 ++++++++++++++++++ packages/api/src/generated/schema.graphql | 16 ++++++++++++++++ packages/api/src/schema.ts | 16 ++++++++++++++++ packages/api/src/services/save_page.ts | 16 ++++++++++------ packages/api/src/utils/parser.ts | 15 ++++++++++++--- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 07b53580a..0c26798f8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1612,6 +1612,21 @@ export enum PageType { Website = 'WEBSITE' } +export type ParseResult = { + byline: Scalars['String']; + content: Scalars['String']; + dir: Scalars['String']; + excerpt: Scalars['String']; + language?: InputMaybe; + length: Scalars['Int']; + previewImage?: InputMaybe; + publishedDate?: InputMaybe; + siteIcon?: InputMaybe; + siteName?: InputMaybe; + textContent: Scalars['String']; + title: Scalars['String']; +}; + export type PreparedDocumentInput = { document: Scalars['String']; pageInfo: PageInfoInput; @@ -2071,6 +2086,7 @@ export type SaveFilterSuccess = { export type SavePageInput = { clientRequestId: Scalars['ID']; originalContent: Scalars['String']; + parseResult?: InputMaybe; source: Scalars['String']; title?: InputMaybe; url: Scalars['String']; @@ -3291,6 +3307,7 @@ export type ResolversTypes = { PageInfo: ResolverTypeWrapper; PageInfoInput: PageInfoInput; PageType: PageType; + ParseResult: ParseResult; PreparedDocumentInput: PreparedDocumentInput; Profile: ResolverTypeWrapper; Query: ResolverTypeWrapper<{}>; @@ -3704,6 +3721,7 @@ export type ResolversParentTypes = { Page: Page; PageInfo: PageInfo; PageInfoInput: PageInfoInput; + ParseResult: ParseResult; PreparedDocumentInput: PreparedDocumentInput; Profile: Profile; Query: {}; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 6036c33fc..7d4df8a16 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1169,6 +1169,21 @@ enum PageType { WEBSITE } +input ParseResult { + byline: String! + content: String! + dir: String! + excerpt: String! + language: String + length: Int! + previewImage: String + publishedDate: Date + siteIcon: String + siteName: String + textContent: String! + title: String! +} + input PreparedDocumentInput { document: String! pageInfo: PageInfoInput! @@ -1501,6 +1516,7 @@ type SaveFilterSuccess { input SavePageInput { clientRequestId: ID! originalContent: String! + parseResult: ParseResult source: String! title: String url: String! diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 081d4b60f..9a17cf019 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -526,12 +526,28 @@ const schema = gql` uploadFileId: ID! } + input ParseResult { + title: String! + byline: String! + dir: String! + content: String! + textContent: String! + length: Int! + excerpt: String! + siteName: String + siteIcon: String + previewImage: String + publishedDate: Date + language: String + } + input SavePageInput { url: String! source: String! clientRequestId: ID! title: String originalContent: String! + parseResult: ParseResult } input SaveUrlInput { diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index a7997969a..de1d18168 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -77,13 +77,17 @@ export const savePage = async ( input: SavePageInput ): Promise => { const [slug, croppedPathname] = createSlug(input.url, input.title) - const parseResult = await parsePreparedContent(input.url, { - document: input.originalContent, - pageInfo: { - title: input.title, - canonicalUrl: input.url, + const parseResult = await parsePreparedContent( + input.url, + { + document: input.originalContent, + pageInfo: { + title: input.title, + canonicalUrl: input.url, + }, }, - }) + input.parseResult + ) const articleToSave = parsedContentToPage({ url: input.url, diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index e19681821..243513667 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -17,8 +17,8 @@ import { v4 as uuid } from 'uuid' import addressparser from 'addressparser' import { preParseContent } from '@omnivore/content-handler' import { - findEmbeddedHighlight, EmbeddedHighlightData, + findEmbeddedHighlight, } from './highlightGenerator' const logger = buildLogger('utils.parse') @@ -174,6 +174,7 @@ const getReadabilityResult = async ( export const parsePreparedContent = async ( url: string, preparedDocument: PreparedDocumentInput, + parseResult?: Readability.ParseResult | null, isNewsletter?: boolean, allowRetry = true ): Promise => { @@ -208,13 +209,21 @@ export const parsePreparedContent = async ( preParsedDom && (dom = preParsedDom) try { - article = await getReadabilityResult(url, document, dom, isNewsletter) + article = + parseResult || + (await getReadabilityResult(url, document, dom, isNewsletter)) if (!article?.textContent && allowRetry) { const newDocument = { ...preparedDocument, document: '' + preparedDocument.document + '', } - return parsePreparedContent(url, newDocument, isNewsletter, false) + return parsePreparedContent( + url, + newDocument, + parseResult, + isNewsletter, + false + ) } // Format code blocks From e22d20972174b2994195cd91b0aa9342c9ba7c00 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 28 Dec 2022 09:16:44 +0800 Subject: [PATCH 3/7] Fix test --- packages/api/src/generated/graphql.ts | 4 ++-- packages/api/src/generated/schema.graphql | 4 ++-- packages/api/src/readability.d.ts | 8 ++++---- packages/api/src/schema.ts | 4 ++-- packages/api/src/services/save_page.ts | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 0c26798f8..a8a6b2f84 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1621,8 +1621,8 @@ export type ParseResult = { length: Scalars['Int']; previewImage?: InputMaybe; publishedDate?: InputMaybe; - siteIcon?: InputMaybe; - siteName?: InputMaybe; + siteIcon: Scalars['String']; + siteName: Scalars['String']; textContent: Scalars['String']; title: Scalars['String']; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 7d4df8a16..980d19cdf 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1178,8 +1178,8 @@ input ParseResult { length: Int! previewImage: String publishedDate: Date - siteIcon: String - siteName: String + siteIcon: String! + siteName: String! textContent: String! title: String! } diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index 2587b114e..cd2504047 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -161,11 +161,11 @@ declare module '@omnivore/readability' { /** Article site icon */ siteIcon: string /** Article preview image */ - previewImage?: string + previewImage?: string | null /** Article published date */ - publishedDate?: Date - dom?: Element - language?: string + publishedDate?: Date | null + dom?: Element | null + language?: string | null } } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 9a17cf019..14abca849 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -534,8 +534,8 @@ const schema = gql` textContent: String! length: Int! excerpt: String! - siteName: String - siteIcon: String + siteName: String! + siteIcon: String! previewImage: String publishedDate: Date language: String diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index de1d18168..5ecc88f18 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -233,8 +233,8 @@ export const parsedContentToPage = ({ }), pageType, hash: uploadFileHash || stringToHash(parsedContent?.content || url), - image: parsedContent?.previewImage, - publishedAt: validatedDate(parsedContent?.publishedDate), + image: parsedContent?.previewImage ?? undefined, + publishedAt: validatedDate(parsedContent?.publishedDate ?? undefined), uploadFileId: uploadFileId, readingProgressPercent: 0, readingProgressAnchorIndex: 0, @@ -242,7 +242,7 @@ export const parsedContentToPage = ({ createdAt: saveTime || new Date(), savedAt: saveTime || new Date(), siteName: parsedContent?.siteName, - language: parsedContent?.language, + language: parsedContent?.language ?? undefined, siteIcon: parsedContent?.siteIcon, wordsCount: wordsCount(parsedContent?.textContent || ''), } From 7c39db207b1fe50a9da474aded74772784474d0e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 28 Dec 2022 10:15:05 +0800 Subject: [PATCH 4/7] Replace createArticle with savePage in puppeteer-parse service --- packages/api/src/generated/graphql.ts | 8 +-- packages/api/src/generated/schema.graphql | 8 +-- packages/api/src/readability.d.ts | 9 ++-- packages/api/src/schema.ts | 8 +-- packages/api/src/services/save_email.ts | 13 +++-- packages/api/src/services/save_page.ts | 6 +-- packages/api/src/utils/parser.ts | 15 +++--- packages/puppeteer-parse/index.js | 65 ++++++++++++++--------- packages/readabilityjs/Readability.js | 1 - 9 files changed, 77 insertions(+), 56 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index a8a6b2f84..575cfd9a8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1613,16 +1613,16 @@ export enum PageType { } export type ParseResult = { - byline: Scalars['String']; + byline?: InputMaybe; content: Scalars['String']; - dir: Scalars['String']; + dir?: InputMaybe; excerpt: Scalars['String']; language?: InputMaybe; length: Scalars['Int']; previewImage?: InputMaybe; publishedDate?: InputMaybe; - siteIcon: Scalars['String']; - siteName: Scalars['String']; + siteIcon?: InputMaybe; + siteName?: InputMaybe; textContent: Scalars['String']; title: Scalars['String']; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 980d19cdf..8d556e138 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1170,16 +1170,16 @@ enum PageType { } input ParseResult { - byline: String! + byline: String content: String! - dir: String! + dir: String excerpt: String! language: String length: Int! previewImage: String publishedDate: Date - siteIcon: String! - siteName: String! + siteIcon: String + siteName: String textContent: String! title: String! } diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index cd2504047..6b7bf0077 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -145,9 +145,9 @@ declare module '@omnivore/readability' { /** Article title */ title: string /** Author metadata */ - byline: string + byline?: string | null /** Content direction */ - dir: string + dir?: string | null /** HTML string of processed article content */ content: string /** non-HTML version of `content` */ @@ -157,14 +157,13 @@ declare module '@omnivore/readability' { /** Article description, or short excerpt from the content */ excerpt: string /** Article site name */ - siteName: string + siteName?: string | null /** Article site icon */ - siteIcon: string + siteIcon?: string | null /** Article preview image */ previewImage?: string | null /** Article published date */ publishedDate?: Date | null - dom?: Element | null language?: string | null } } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 14abca849..0128f88fb 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -528,14 +528,14 @@ const schema = gql` input ParseResult { title: String! - byline: String! - dir: String! + byline: String + dir: String content: String! textContent: String! length: Int! excerpt: String! - siteName: String! - siteIcon: String! + siteName: String + siteIcon: String previewImage: String publishedDate: Date language: String diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index da623b7a0..19e2ec024 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -41,6 +41,7 @@ export const saveEmail = async ( // can leave this empty for now }, }, + null, true ) const content = parseResult.parsedContent?.content || input.originalContent @@ -62,15 +63,21 @@ export const saveEmail = async ( }), pageType: parseResult.pageType, hash: stringToHash(content), - image: metadata?.previewImage || parseResult.parsedContent?.previewImage, - publishedAt: validatedDate(parseResult.parsedContent?.publishedDate), + image: + metadata?.previewImage || + parseResult.parsedContent?.previewImage || + undefined, + publishedAt: validatedDate( + parseResult.parsedContent?.publishedDate ?? undefined + ), createdAt: new Date(), savedAt: new Date(), readingProgressAnchorIndex: 0, readingProgressPercent: 0, subscription: input.author, state: ArticleSavingRequestStatus.Succeeded, - siteIcon: parseResult.parsedContent?.siteIcon, + siteIcon: parseResult.parsedContent?.siteIcon ?? undefined, + siteName: parseResult.parsedContent?.siteName ?? undefined, } const page = await getPageByParam({ diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 5ecc88f18..d3270ce6e 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -226,7 +226,7 @@ export const parsedContentToPage = ({ croppedPathname || parsedContent?.siteName || url, - author: parsedContent?.byline, + author: parsedContent?.byline ?? undefined, url: normalizeUrl(canonicalUrl || url, { stripHash: true, stripWWW: false, @@ -241,9 +241,9 @@ export const parsedContentToPage = ({ state: ArticleSavingRequestStatus.Succeeded, createdAt: saveTime || new Date(), savedAt: saveTime || new Date(), - siteName: parsedContent?.siteName, + siteName: parsedContent?.siteName ?? undefined, language: parsedContent?.language ?? undefined, - siteIcon: parsedContent?.siteIcon, + siteIcon: parsedContent?.siteIcon ?? undefined, wordsCount: wordsCount(parsedContent?.textContent || ''), } } diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 243513667..b5739beba 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -229,8 +229,9 @@ export const parsePreparedContent = async ( // Format code blocks // TODO: we probably want to move this type of thing // to the handlers, and have some concept of postHandle - if (article?.dom) { - const codeBlocks = article.dom.querySelectorAll('code') + if (article?.content) { + const articleDom = parseHTML(article.content).document + const codeBlocks = articleDom.querySelectorAll('code') if (codeBlocks.length > 0) { codeBlocks.forEach((e) => { if (e.textContent) { @@ -246,12 +247,10 @@ export const parsePreparedContent = async ( e.replaceWith(code) } }) - article.content = article.dom.outerHTML + article.content = articleDom.documentElement.outerHTML } - if (article?.dom) { - highlightData = findEmbeddedHighlight(article?.dom) - } + highlightData = findEmbeddedHighlight(articleDom.documentElement) const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ 'omnivore-highlight-id', @@ -260,7 +259,7 @@ export const parsePreparedContent = async ( ] // Get the top level element? - const pageNode = article.dom.firstElementChild as HTMLElement + const pageNode = articleDom.firstElementChild as HTMLElement const nodesToVisitStack: [HTMLElement] = [pageNode] const visitedNodeList = [] @@ -290,7 +289,7 @@ export const parsePreparedContent = async ( node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) }) - article.content = article.dom.outerHTML + article.content = articleDom.documentElement.outerHTML } const newWindow = parseHTML('') diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index ae2dd942b..73950791a 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -23,7 +23,7 @@ puppeteer.use(StealthPlugin()); // Add adblocker plugin to block all ads and trackers (saves bandwidth) const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); -puppeteer.use(AdblockerPlugin({ blockTrackers: true })); +// puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; @@ -200,6 +200,35 @@ const sendCreateArticleMutation = async (userId, input) => { return response.data.data.createArticle; }; +const sendSavePageMutation = async (userId, input) => { + const data = JSON.stringify({ + query: `mutation SavePage ($input: SavePageInput!){ + savePage(input:$input){ + ... on SaveSuccess{ + url + clientRequestId + } + ... on SaveError{ + errorCodes + } + } + }`, + variables: { + input: Object.assign({}, input , { source: 'puppeteer-parse' }), + }, + }); + + const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); + const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); + return response.data.data.savePage; +}; + const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => { return sendCreateArticleMutation(userId, { url: encodeURI(url), @@ -283,18 +312,12 @@ async function fetchContent(req, res) { const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; - const apiResponse = await sendCreateArticleMutation(userId, { + const apiResponse = await sendSavePageMutation(userId, { url: finalUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title, - canonicalUrl: finalUrl, - }, - }, - skipParsing: !!readabilityResult, - readabilityResult, + clientRequestId: articleSavingRequestId, + title, + originalContent: content, + parseResult: readabilityResult, }); logRecord.totalTime = Date.now() - functionStartTime; @@ -312,18 +335,12 @@ async function fetchContent(req, res) { const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; - const apiResponse = await sendCreateArticleMutation(userId, { - url: sbUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title: sbResult.title, - canonicalUrl: sbUrl, - }, - }, - skipParsing: !!readabilityResult, - readabilityResult, + const apiResponse = await sendSavePageMutation(userId, { + url: finalUrl, + clientRequestId: articleSavingRequestId, + title, + originalContent: content, + parseResult: readabilityResult, }); logRecord.totalTime = Date.now() - functionStartTime; diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 437e878f5..f84e47501 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -2998,7 +2998,6 @@ Readability.prototype = { siteIcon: metadata.siteIcon, previewImage: metadata.previewImage, publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate, - dom: articleContent, language: this._getLanguage(metadata.locale || this._languageCode), }; } From a5f5e6fbdb6f531a93d47d35350e33edb8b91bd6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 28 Dec 2022 11:51:11 +0800 Subject: [PATCH 5/7] Fix docker build --- packages/content-fetch/Dockerfile | 1 + packages/content-fetch/Dockerfile-gcf | 1 + packages/puppeteer-parse/index.js | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile index 9ca9f7dee..0e47309c2 100644 --- a/packages/content-fetch/Dockerfile +++ b/packages/content-fetch/Dockerfile @@ -37,6 +37,7 @@ RUN yarn install --pure-lockfile ADD /packages/content-fetch ./packages/content-fetch ADD /packages/content-handler ./packages/content-handler ADD /packages/puppeteer-parse ./packages/puppeteer-parse +ADD /packages/readabilityjs ./packages/readabilityjs RUN yarn workspace @omnivore/content-handler build # After building, fetch the production dependencies diff --git a/packages/content-fetch/Dockerfile-gcf b/packages/content-fetch/Dockerfile-gcf index 8c355485f..4e7433326 100644 --- a/packages/content-fetch/Dockerfile-gcf +++ b/packages/content-fetch/Dockerfile-gcf @@ -38,6 +38,7 @@ RUN yarn install --pure-lockfile ADD /packages/content-handler ./packages/content-handler ADD /packages/puppeteer-parse ./packages/puppeteer-parse ADD /packages/content-fetch ./packages/content-fetch +ADD /packages/readabilityjs ./packages/readabilityjs RUN yarn workspace @omnivore/content-handler build # After building, fetch the production dependencies diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 73950791a..d03c5f3fb 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -23,7 +23,7 @@ puppeteer.use(StealthPlugin()); // Add adblocker plugin to block all ads and trackers (saves bandwidth) const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); -// puppeteer.use(AdblockerPlugin({ blockTrackers: true })); +puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; From 6cd6994affd1fc1548ab0f736adc71056d7341c8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 28 Dec 2022 15:28:27 +0800 Subject: [PATCH 6/7] Fix docker image --- packages/content-fetch/Dockerfile | 6 +++++- packages/content-fetch/Dockerfile-gcf | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/content-fetch/Dockerfile b/packages/content-fetch/Dockerfile index 0e47309c2..1c5f4830f 100644 --- a/packages/content-fetch/Dockerfile +++ b/packages/content-fetch/Dockerfile @@ -9,7 +9,10 @@ RUN apk add --no-cache \ ca-certificates \ ttf-freefont \ nodejs \ - yarn + yarn \ + g++ \ + make \ + python3 # Add user so we don't need --no-sandbox. RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ @@ -29,6 +32,7 @@ COPY tsconfig.json . COPY .prettierrc . COPY .eslintrc . +COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json COPY /packages/content-handler/package.json ./packages/content-handler/package.json COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json diff --git a/packages/content-fetch/Dockerfile-gcf b/packages/content-fetch/Dockerfile-gcf index 4e7433326..67a4e8ee1 100644 --- a/packages/content-fetch/Dockerfile-gcf +++ b/packages/content-fetch/Dockerfile-gcf @@ -9,7 +9,10 @@ RUN apk add --no-cache \ ca-certificates \ ttf-freefont \ nodejs \ - yarn + yarn \ + g++ \ + make \ + python3 # Add user so we don't need --no-sandbox. RUN addgroup -S pptruser && adduser -S -g pptruser pptruser \ @@ -30,6 +33,7 @@ COPY tsconfig.json . COPY .prettierrc . COPY .eslintrc . +COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json COPY /packages/content-handler/package.json ./packages/content-handler/package.json COPY /packages/puppeteer-parse/package.json ./packages/puppeteer-parse/package.json From 8dc4c48f87df7c8e1fe5537bd6d96681fca85b08 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 28 Dec 2022 16:51:23 +0800 Subject: [PATCH 7/7] Add createImageProxyUrl callback to the readability in puppeteer-parse service --- packages/puppeteer-parse/index.js | 99 +++++++++++++++++++++++++-- packages/puppeteer-parse/package.json | 5 +- yarn.lock | 12 +++- 3 files changed, 110 insertions(+), 6 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index d03c5f3fb..f2ec56f42 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -3,6 +3,9 @@ /* eslint-disable @typescript-eslint/explicit-function-return-type */ /* eslint-disable @typescript-eslint/no-var-requires */ /* eslint-disable @typescript-eslint/no-require-imports */ +const { encode } = require("urlsafe-base64"); +const crypto = require("crypto"); + const Url = require('url'); // const puppeteer = require('puppeteer-extra'); const axios = require('axios'); @@ -23,6 +26,7 @@ puppeteer.use(StealthPlugin()); // Add adblocker plugin to block all ads and trackers (saves bandwidth) const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); +const createDOMPurify = require("dompurify"); puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); @@ -782,10 +786,97 @@ async function preview(req, res) { return res.redirect(`${process.env.PREVIEW_IMAGE_CDN_ORIGIN}/${destination}`); } -async function getReadabilityResult(url, domContent) { - const document = parseHTML(domContent).document; - const readability = new Readability(document, { url }); - return readability.parse(); +const DOM_PURIFY_CONFIG = { + ADD_TAGS: ['iframe'], + ADD_ATTR: ['allow', 'allowfullscreen', 'frameborder', 'scrolling'], + FORBID_ATTR: [ + 'data-ml-dynamic', + 'data-ml-dynamic-type', + 'data-orig-url', + 'data-ml-id', + 'data-ml', + 'data-xid', + 'data-feature', + ], +} + +function domPurifySanitizeHook(node, data) { + if (data.tagName === 'iframe') { + const urlRegex = /^(https?:)?\/\/www\.youtube(-nocookie)?\.com\/embed\//i + const src = node.getAttribute('src') || '' + const dataSrc = node.getAttribute('data-src') || '' + + if (src && urlRegex.test(src)) { + return + } + + if (dataSrc && urlRegex.test(dataSrc)) { + node.setAttribute('src', dataSrc) + return + } + + node.parentNode?.removeChild(node) + } +} + +function getPurifiedContent(html) { + const newWindow = parseHTML('') + const DOMPurify = createDOMPurify(newWindow) + DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) + const clean = DOMPurify.sanitize(html, DOM_PURIFY_CONFIG) + return parseHTML(clean).document +} + +function signImageProxyUrl(url) { + return encode( + crypto.createHmac('sha256', process.env.IMAGE_PROXY_SECRET).update(url).digest() + ) +} + +function createImageProxyUrl(url, width = 0, height = 0) { + if (!process.env.IMAGE_PROXY_URL || !process.env.IMAGE_PROXY_SECRET) { + return url + } + + const urlWithOptions = `${url}#${width}x${height}` + const signature = signImageProxyUrl(urlWithOptions) + + return `${process.env.IMAGE_PROXY_URL}/${width}x${height},s${signature}/${url}` +} + +async function getReadabilityResult(url, document) { + // First attempt to read the article as is. + // if that fails attempt to purify then read + const sources = [ + () => { + return document + }, + () => { + return getPurifiedContent(document) + }, + ] + + for (const source of sources) { + const document = source() + if (!document) { + continue + } + + try { + const article = await new Readability(document, { + createImageProxyUrl, + url, + }).parse() + + if (article) { + return article + } + } catch (error) { + console.log('parsing error for url', url, error) + } + } + + return null } module.exports = { diff --git a/packages/puppeteer-parse/package.json b/packages/puppeteer-parse/package.json index 8798377a9..540070b50 100644 --- a/packages/puppeteer-parse/package.json +++ b/packages/puppeteer-parse/package.json @@ -8,13 +8,16 @@ "@omnivore/content-handler": "1.0.0", "@omnivore/readability": "1.0.0", "axios": "^0.27.2", + "crypto": "^1.0.1", + "dompurify": "^2.4.1", "jsonwebtoken": "^8.5.1", "linkedom": "^0.14.9", "puppeteer-core": "^16.1.0", "puppeteer-extra": "^3.3.4", "puppeteer-extra-plugin-adblocker": "^2.13.5", "puppeteer-extra-plugin-stealth": "^2.11.1", - "underscore": "^1.13.4" + "underscore": "^1.13.4", + "urlsafe-base64": "^1.0.0" }, "devDependencies": { "chai": "^4.3.6", diff --git a/yarn.lock b/yarn.lock index 2395d7803..c4990abaa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12251,6 +12251,11 @@ crypto-random-string@^2.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-2.0.0.tgz#ef2a7a966ec11083388369baa02ebead229b30d5" integrity sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA== +crypto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/crypto/-/crypto-1.0.1.tgz#2af1b7cad8175d24c8a1b0778255794a21803037" + integrity sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig== + css-loader@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.6.0.tgz#2e4b2c7e6e2d27f8c8f28f61bffcd2e6c91ef645" @@ -13055,6 +13060,11 @@ dompurify@^2.0.17: resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.3.8.tgz#224fe9ae57d7ebd9a1ae1ac18c1c1ca3f532226f" integrity sha512-eVhaWoVibIzqdGYjwsBWodIQIaXFSB+cKDf4cfxLMsK0xiud6SE+/WCVx/Xw/UwQsa4cS3T2eITcdtmTg2UKcw== +dompurify@^2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-2.4.1.tgz#f9cb1a275fde9af6f2d0a2644ef648dd6847b631" + integrity sha512-ewwFzHzrrneRjxzmK6oVz/rZn9VWspGFRDb4/rRtIsM1n36t9AKma/ye8syCpcw+XJ25kOK/hOG7t1j2I2yBqA== + domutils@^2.0.0, domutils@^2.5.2: version "2.7.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-2.7.0.tgz#8ebaf0c41ebafcf55b0b72ec31c56323712c5442" @@ -26280,7 +26290,7 @@ url@^0.11.0: urlsafe-base64@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/urlsafe-base64/-/urlsafe-base64-1.0.0.tgz#23f89069a6c62f46cf3a1d3b00169cefb90be0c6" - integrity sha1-I/iQaabGL0bPOh07ABac77kL4MY= + integrity sha512-RtuPeMy7c1UrHwproMZN9gN6kiZ0SvJwRaEzwZY0j9MypEkFqyBaKv176jvlPtg58Zh36bOkS0NFABXMHvvGCA== use-callback-ref@^1.2.3: version "1.2.5"