diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index a8a6b2f84..575cfd9a8 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -1613,16 +1613,16 @@ export enum PageType { } export type ParseResult = { - byline: Scalars['String']; + byline?: InputMaybe; content: Scalars['String']; - dir: Scalars['String']; + dir?: InputMaybe; excerpt: Scalars['String']; language?: InputMaybe; length: Scalars['Int']; previewImage?: InputMaybe; publishedDate?: InputMaybe; - siteIcon: Scalars['String']; - siteName: Scalars['String']; + siteIcon?: InputMaybe; + siteName?: InputMaybe; textContent: Scalars['String']; title: Scalars['String']; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 980d19cdf..8d556e138 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1170,16 +1170,16 @@ enum PageType { } input ParseResult { - byline: String! + byline: String content: String! - dir: String! + dir: String excerpt: String! language: String length: Int! previewImage: String publishedDate: Date - siteIcon: String! - siteName: String! + siteIcon: String + siteName: String textContent: String! title: String! } diff --git a/packages/api/src/readability.d.ts b/packages/api/src/readability.d.ts index cd2504047..6b7bf0077 100644 --- a/packages/api/src/readability.d.ts +++ b/packages/api/src/readability.d.ts @@ -145,9 +145,9 @@ declare module '@omnivore/readability' { /** Article title */ title: string /** Author metadata */ - byline: string + byline?: string | null /** Content direction */ - dir: string + dir?: string | null /** HTML string of processed article content */ content: string /** non-HTML version of `content` */ @@ -157,14 +157,13 @@ declare module '@omnivore/readability' { /** Article description, or short excerpt from the content */ excerpt: string /** Article site name */ - siteName: string + siteName?: string | null /** Article site icon */ - siteIcon: string + siteIcon?: string | null /** Article preview image */ previewImage?: string | null /** Article published date */ publishedDate?: Date | null - dom?: Element | null language?: string | null } } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 14abca849..0128f88fb 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -528,14 +528,14 @@ const schema = gql` input ParseResult { title: String! - byline: String! - dir: String! + byline: String + dir: String content: String! textContent: String! length: Int! excerpt: String! - siteName: String! - siteIcon: String! + siteName: String + siteIcon: String previewImage: String publishedDate: Date language: String diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index da623b7a0..19e2ec024 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -41,6 +41,7 @@ export const saveEmail = async ( // can leave this empty for now }, }, + null, true ) const content = parseResult.parsedContent?.content || input.originalContent @@ -62,15 +63,21 @@ export const saveEmail = async ( }), pageType: parseResult.pageType, hash: stringToHash(content), - image: metadata?.previewImage || parseResult.parsedContent?.previewImage, - publishedAt: validatedDate(parseResult.parsedContent?.publishedDate), + image: + metadata?.previewImage || + parseResult.parsedContent?.previewImage || + undefined, + publishedAt: validatedDate( + parseResult.parsedContent?.publishedDate ?? undefined + ), createdAt: new Date(), savedAt: new Date(), readingProgressAnchorIndex: 0, readingProgressPercent: 0, subscription: input.author, state: ArticleSavingRequestStatus.Succeeded, - siteIcon: parseResult.parsedContent?.siteIcon, + siteIcon: parseResult.parsedContent?.siteIcon ?? undefined, + siteName: parseResult.parsedContent?.siteName ?? undefined, } const page = await getPageByParam({ diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 5ecc88f18..d3270ce6e 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -226,7 +226,7 @@ export const parsedContentToPage = ({ croppedPathname || parsedContent?.siteName || url, - author: parsedContent?.byline, + author: parsedContent?.byline ?? undefined, url: normalizeUrl(canonicalUrl || url, { stripHash: true, stripWWW: false, @@ -241,9 +241,9 @@ export const parsedContentToPage = ({ state: ArticleSavingRequestStatus.Succeeded, createdAt: saveTime || new Date(), savedAt: saveTime || new Date(), - siteName: parsedContent?.siteName, + siteName: parsedContent?.siteName ?? undefined, language: parsedContent?.language ?? undefined, - siteIcon: parsedContent?.siteIcon, + siteIcon: parsedContent?.siteIcon ?? undefined, wordsCount: wordsCount(parsedContent?.textContent || ''), } } diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index 243513667..b5739beba 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -229,8 +229,9 @@ export const parsePreparedContent = async ( // Format code blocks // TODO: we probably want to move this type of thing // to the handlers, and have some concept of postHandle - if (article?.dom) { - const codeBlocks = article.dom.querySelectorAll('code') + if (article?.content) { + const articleDom = parseHTML(article.content).document + const codeBlocks = articleDom.querySelectorAll('code') if (codeBlocks.length > 0) { codeBlocks.forEach((e) => { if (e.textContent) { @@ -246,12 +247,10 @@ export const parsePreparedContent = async ( e.replaceWith(code) } }) - article.content = article.dom.outerHTML + article.content = articleDom.documentElement.outerHTML } - if (article?.dom) { - highlightData = findEmbeddedHighlight(article?.dom) - } + highlightData = findEmbeddedHighlight(articleDom.documentElement) const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [ 'omnivore-highlight-id', @@ -260,7 +259,7 @@ export const parsePreparedContent = async ( ] // Get the top level element? - const pageNode = article.dom.firstElementChild as HTMLElement + const pageNode = articleDom.firstElementChild as HTMLElement const nodesToVisitStack: [HTMLElement] = [pageNode] const visitedNodeList = [] @@ -290,7 +289,7 @@ export const parsePreparedContent = async ( node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString()) }) - article.content = article.dom.outerHTML + article.content = articleDom.documentElement.outerHTML } const newWindow = parseHTML('') diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index ae2dd942b..73950791a 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -23,7 +23,7 @@ puppeteer.use(StealthPlugin()); // Add adblocker plugin to block all ads and trackers (saves bandwidth) const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); -puppeteer.use(AdblockerPlugin({ blockTrackers: true })); +// puppeteer.use(AdblockerPlugin({ blockTrackers: true })); const storage = new Storage(); const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : []; @@ -200,6 +200,35 @@ const sendCreateArticleMutation = async (userId, input) => { return response.data.data.createArticle; }; +const sendSavePageMutation = async (userId, input) => { + const data = JSON.stringify({ + query: `mutation SavePage ($input: SavePageInput!){ + savePage(input:$input){ + ... on SaveSuccess{ + url + clientRequestId + } + ... on SaveError{ + errorCodes + } + } + }`, + variables: { + input: Object.assign({}, input , { source: 'puppeteer-parse' }), + }, + }); + + const auth = await signToken({ uid: userId }, process.env.JWT_SECRET); + const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data, + { + headers: { + Cookie: `auth=${auth};`, + 'Content-Type': 'application/json', + }, + }); + return response.data.data.savePage; +}; + const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => { return sendCreateArticleMutation(userId, { url: encodeURI(url), @@ -283,18 +312,12 @@ async function fetchContent(req, res) { const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; - const apiResponse = await sendCreateArticleMutation(userId, { + const apiResponse = await sendSavePageMutation(userId, { url: finalUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title, - canonicalUrl: finalUrl, - }, - }, - skipParsing: !!readabilityResult, - readabilityResult, + clientRequestId: articleSavingRequestId, + title, + originalContent: content, + parseResult: readabilityResult, }); logRecord.totalTime = Date.now() - functionStartTime; @@ -312,18 +335,12 @@ async function fetchContent(req, res) { const readabilityResult = content ? (await getReadabilityResult(url, content)) : null; - const apiResponse = await sendCreateArticleMutation(userId, { - url: sbUrl, - articleSavingRequestId, - preparedDocument: { - document: content, - pageInfo: { - title: sbResult.title, - canonicalUrl: sbUrl, - }, - }, - skipParsing: !!readabilityResult, - readabilityResult, + const apiResponse = await sendSavePageMutation(userId, { + url: finalUrl, + clientRequestId: articleSavingRequestId, + title, + originalContent: content, + parseResult: readabilityResult, }); logRecord.totalTime = Date.now() - functionStartTime; diff --git a/packages/readabilityjs/Readability.js b/packages/readabilityjs/Readability.js index 437e878f5..f84e47501 100644 --- a/packages/readabilityjs/Readability.js +++ b/packages/readabilityjs/Readability.js @@ -2998,7 +2998,6 @@ Readability.prototype = { siteIcon: metadata.siteIcon, previewImage: metadata.previewImage, publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate, - dom: articleContent, language: this._getLanguage(metadata.locale || this._languageCode), }; }