prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1648 from omnivore-app/fix/puppeteer-parse

Browse source 
fix/puppeteer parse
This commit is contained in:
Hongbo Wu 2023-01-13 10:59:09 +08:00 committed by GitHub
commit 6572f718c9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 42 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/api/src/services/subscriptions.ts
View file

@ -50,7 +50,7 @@ export const saveSubscription = async ({
const result = await getRepository(Subscription).upsert(
{
name,
newsletterEmail,
newsletterEmail: { id: newsletterEmail.id },
user: { id: userId },
status: SubscriptionStatus.Active,
unsubscribeHttpUrl,

19
packages/api/src/utils/parser.ts
View file

@ -184,7 +184,8 @@ export const parsePreparedContent = async (
labels: { source: 'parsePreparedContent' },
}
let article = null
// If we have a parse result, use it
let article = parseResult || null
let highlightData = undefined
const { document, pageInfo } = preparedDocument
@ -205,14 +206,16 @@ export const parsePreparedContent = async (
let dom = parseHTML(document).document
// preParse content
const preParsedDom = await preParseContent(url, dom)
preParsedDom && (dom = preParsedDom)
try {
article =
parseResult ||
(await getReadabilityResult(url, document, dom, isNewsletter))
if (!article) {
// Attempt to parse the article
// preParse content
const preParsedDom = await preParseContent(url, dom)
preParsedDom && (dom = preParsedDom)
article = await getReadabilityResult(url, document, dom, isNewsletter)
}
if (!article?.textContent && allowRetry) {
const newDocument = {
...preparedDocument,

2
packages/api/test/db.ts
View file

@ -203,7 +203,7 @@ export const createTestSubscription = async (
user,
name,
status: SubscriptionStatus.Active,
newsletterEmail,
newsletterEmail: { id: newsletterEmail.id },
})
}

6
packages/content-handler/src/websites/twitter-handler.ts
View file

@ -274,7 +274,7 @@ const getTweetIds = async (
}
window.scrollBy(0, distance)
await waitFor(100)
await waitFor(500)
currentHeight += distance
}
@ -364,6 +364,7 @@ export class TwitterHandler extends ContentHandler {
`
const content = `
<html>
<head>
<meta property="og:image" content="${authorImage}" />
<meta property="og:image:secure_url" content="${authorImage}" />
@ -375,7 +376,8 @@ export class TwitterHandler extends ContentHandler {
${tweetsContent}
${tweetUrl}
</div>
</body>`
</body>
</html>`
return { content, url, title }
}

28
packages/puppeteer-parse/index.js
View file

@ -15,7 +15,7 @@ const signToken = promisify(jwt.sign);
const os = require('os');
const { Storage } = require('@google-cloud/storage');
const { parseHTML } = require('linkedom');
const { preHandleContent } = require("@omnivore/content-handler");
const { preHandleContent, preParseContent } = require("@omnivore/content-handler");
const { Readability } = require("@omnivore/readability");
const puppeteer = require('puppeteer-extra');
@ -314,7 +314,18 @@ async function fetchContent(req, res) {
logRecord.fetchContentTime = Date.now() - functionStartTime;
const readabilityResult = content ? (await getReadabilityResult(url, content)) : null;
let readabilityResult = null;
if (content) {
let document = parseHTML(content).document;
// preParse content
const preParsedDom = await preParseContent(url, document)
if (preParsedDom) {
document = preParsedDom
}
readabilityResult = await getReadabilityResult(url, document);
}
const apiResponse = await sendSavePageMutation(userId, {
url: finalUrl,
@ -337,7 +348,18 @@ async function fetchContent(req, res) {
const content = sbResult.domContent;
logRecord.fetchContentTime = Date.now() - functionStartTime;
const readabilityResult = content ? (await getReadabilityResult(url, content)) : null;
let readabilityResult = null;
if (content) {
let document = parseHTML(content).document;
// preParse content
const preParsedDom = await preParseContent(sbUrl, document)
if (preParsedDom) {
document = preParsedDom
}
readabilityResult = await getReadabilityResult(url, document);
}
const apiResponse = await sendSavePageMutation(userId, {
url: finalUrl,