From 7bfd36e308cf6d44ffba917007869a643231c77b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 12 Jan 2023 18:33:47 +0800 Subject: [PATCH 1/4] Fix readability not run in puppeteer-parse --- packages/api/src/utils/parser.ts | 19 +++++++++++-------- packages/puppeteer-parse/index.js | 15 +++++++++++++-- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 431f6619e..8fd3a4d30 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -184,7 +184,8 @@ export const parsePreparedContent = async ( labels: { source: 'parsePreparedContent' }, } - let article = null + // If we have a parse result, use it + let article = parseResult || null let highlightData = undefined const { document, pageInfo } = preparedDocument @@ -205,14 +206,16 @@ export const parsePreparedContent = async ( let dom = parseHTML(document).document - // preParse content - const preParsedDom = await preParseContent(url, dom) - preParsedDom && (dom = preParsedDom) - try { - article = - parseResult || - (await getReadabilityResult(url, document, dom, isNewsletter)) + if (!article) { + // Attempt to parse the article + // preParse content + const preParsedDom = await preParseContent(url, dom) + preParsedDom && (dom = preParsedDom) + + article = await getReadabilityResult(url, document, dom, isNewsletter) + } + if (!article?.textContent && allowRetry) { const newDocument = { ...preparedDocument, diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 0ee12be6e..2eb417901 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -15,7 +15,7 @@ const signToken = promisify(jwt.sign); const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); -const { preHandleContent } = require("@omnivore/content-handler"); +const { preHandleContent, preParseContent } = require("@omnivore/content-handler"); const { Readability } = require("@omnivore/readability"); const puppeteer = require('puppeteer-extra'); @@ -314,7 +314,18 @@ async function fetchContent(req, res) { logRecord.fetchContentTime = Date.now() - functionStartTime; - const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; + let readabilityResult = null; + if (content) { + let document = parseHTML(content).document; + + // preParse content + const preParsedDom = await preParseContent(url, document) + if (preParsedDom) { + document = preParsedDom + } + + readabilityResult = await getReadabilityResult(url, document); + } const apiResponse = await sendSavePageMutation(userId, { url: finalUrl, From 145a8b973060e3223ddfa99fdaf3d8581f1f5317 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 12 Jan 2023 22:03:29 +0800 Subject: [PATCH 2/4] Fix last two tweets not saved in a long thread by waiting for document loaded after scrolling --- .../content-handler/src/websites/twitter-handler.ts | 6 ++++-- packages/puppeteer-parse/index.js | 13 ++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 46e635980..c745f6d66 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -274,7 +274,7 @@ const getTweetIds = async ( } window.scrollBy(0, distance) - await waitFor(100) + await waitFor(500) currentHeight += distance } @@ -364,6 +364,7 @@ export class TwitterHandler extends ContentHandler { ` const content = ` + @@ -375,7 +376,8 @@ export class TwitterHandler extends ContentHandler { ${tweetsContent} ${tweetUrl} - ` + +` return { content, url, title } } diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 2eb417901..fa25b4732 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -348,7 +348,18 @@ async function fetchContent(req, res) { const content = sbResult.domContent; logRecord.fetchContentTime = Date.now() - functionStartTime; - const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; + let readabilityResult = null; + if (content) { + let document = parseHTML(content).document; + + // preParse content + const preParsedDom = await preParseContent(sbUrl, document) + if (preParsedDom) { + document = preParsedDom + } + + readabilityResult = await getReadabilityResult(url, document); + } const apiResponse = await sendSavePageMutation(userId, { url: finalUrl, From c647aabeaaddb7ed6ba29b2a082111887ed1e6dc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 12 Jan 2023 22:15:42 +0800 Subject: [PATCH 3/4] Save newsletter_email_id in subscription instead of the whole entity --- packages/api/src/services/subscriptions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 23de44453..edf013f19 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -50,7 +50,7 @@ export const saveSubscription = async ({ const result = await getRepository(Subscription).upsert( { name, - newsletterEmail, + newsletterEmail: { id: newsletterEmail.id }, user: { id: userId }, status: SubscriptionStatus.Active, unsubscribeHttpUrl, From c4a5cbe4e2e1df7c09d6db28de421fbb2c683ef7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 12 Jan 2023 22:16:00 +0800 Subject: [PATCH 4/4] Save newsletter_email_id in subscription instead of the whole entity --- packages/api/test/db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/test/db.ts b/packages/api/test/db.ts index fd4386bfc..89e9a66ab 100644 --- a/packages/api/test/db.ts +++ b/packages/api/test/db.ts @@ -203,7 +203,7 @@ export const createTestSubscription = async ( user, name, status: SubscriptionStatus.Active, - newsletterEmail, + newsletterEmail: { id: newsletterEmail.id }, }) }