diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 23de44453..edf013f19 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -50,7 +50,7 @@ export const saveSubscription = async ({ const result = await getRepository(Subscription).upsert( { name, - newsletterEmail, + newsletterEmail: { id: newsletterEmail.id }, user: { id: userId }, status: SubscriptionStatus.Active, unsubscribeHttpUrl, diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 431f6619e..8fd3a4d30 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -184,7 +184,8 @@ export const parsePreparedContent = async ( labels: { source: 'parsePreparedContent' }, } - let article = null + // If we have a parse result, use it + let article = parseResult || null let highlightData = undefined const { document, pageInfo } = preparedDocument @@ -205,14 +206,16 @@ export const parsePreparedContent = async ( let dom = parseHTML(document).document - // preParse content - const preParsedDom = await preParseContent(url, dom) - preParsedDom && (dom = preParsedDom) - try { - article = - parseResult || - (await getReadabilityResult(url, document, dom, isNewsletter)) + if (!article) { + // Attempt to parse the article + // preParse content + const preParsedDom = await preParseContent(url, dom) + preParsedDom && (dom = preParsedDom) + + article = await getReadabilityResult(url, document, dom, isNewsletter) + } + if (!article?.textContent && allowRetry) { const newDocument = { ...preparedDocument, diff --git a/packages/api/test/db.ts b/packages/api/test/db.ts index fd4386bfc..89e9a66ab 100644 --- a/packages/api/test/db.ts +++ b/packages/api/test/db.ts @@ -203,7 +203,7 @@ export const createTestSubscription = async ( user, name, status: SubscriptionStatus.Active, - newsletterEmail, + newsletterEmail: { id: newsletterEmail.id }, }) } diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts index 46e635980..c745f6d66 100644 --- a/packages/content-handler/src/websites/twitter-handler.ts +++ b/packages/content-handler/src/websites/twitter-handler.ts @@ -274,7 +274,7 @@ const getTweetIds = async ( } window.scrollBy(0, distance) - await waitFor(100) + await waitFor(500) currentHeight += distance } @@ -364,6 +364,7 @@ export class TwitterHandler extends ContentHandler { ` const content = ` + @@ -375,7 +376,8 @@ export class TwitterHandler extends ContentHandler { ${tweetsContent} ${tweetUrl} - ` + +` return { content, url, title } } diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 0ee12be6e..fa25b4732 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -15,7 +15,7 @@ const signToken = promisify(jwt.sign); const os = require('os'); const { Storage } = require('@google-cloud/storage'); const { parseHTML } = require('linkedom'); -const { preHandleContent } = require("@omnivore/content-handler"); +const { preHandleContent, preParseContent } = require("@omnivore/content-handler"); const { Readability } = require("@omnivore/readability"); const puppeteer = require('puppeteer-extra'); @@ -314,7 +314,18 @@ async function fetchContent(req, res) { logRecord.fetchContentTime = Date.now() - functionStartTime; - const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; + let readabilityResult = null; + if (content) { + let document = parseHTML(content).document; + + // preParse content + const preParsedDom = await preParseContent(url, document) + if (preParsedDom) { + document = preParsedDom + } + + readabilityResult = await getReadabilityResult(url, document); + } const apiResponse = await sendSavePageMutation(userId, { url: finalUrl, @@ -337,7 +348,18 @@ async function fetchContent(req, res) { const content = sbResult.domContent; logRecord.fetchContentTime = Date.now() - functionStartTime; - const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; + let readabilityResult = null; + if (content) { + let document = parseHTML(content).document; + + // preParse content + const preParsedDom = await preParseContent(sbUrl, document) + if (preParsedDom) { + document = preParsedDom + } + + readabilityResult = await getReadabilityResult(url, document); + } const apiResponse = await sendSavePageMutation(userId, { url: finalUrl,