Replace createArticle with savePage in puppeteer-parse service

packages/api/src/generated/graphql.ts

@@ -1613,16 +1613,16 @@ export enum PageType {
 }
 
 export type ParseResult = {
-  byline: Scalars['String'];
+  byline?: InputMaybe<Scalars['String']>;
   content: Scalars['String'];
-  dir: Scalars['String'];
+  dir?: InputMaybe<Scalars['String']>;
   excerpt: Scalars['String'];
   language?: InputMaybe<Scalars['String']>;
   length: Scalars['Int'];
   previewImage?: InputMaybe<Scalars['String']>;
   publishedDate?: InputMaybe<Scalars['Date']>;
-  siteIcon: Scalars['String'];
-  siteName: Scalars['String'];
+  siteIcon?: InputMaybe<Scalars['String']>;
+  siteName?: InputMaybe<Scalars['String']>;
   textContent: Scalars['String'];
   title: Scalars['String'];
 };

packages/api/src/generated/schema.graphql

@@ -1170,16 +1170,16 @@ enum PageType {
 }
 
 input ParseResult {
-  byline: String!
+  byline: String
   content: String!
-  dir: String!
+  dir: String
   excerpt: String!
   language: String
   length: Int!
   previewImage: String
   publishedDate: Date
-  siteIcon: String!
-  siteName: String!
+  siteIcon: String
+  siteName: String
   textContent: String!
   title: String!
 }

packages/api/src/readability.d.ts

@@ -145,9 +145,9 @@ declare module '@omnivore/readability' {
       /** Article title */
       title: string
       /** Author metadata */
-      byline: string
+      byline?: string | null
       /** Content direction */
-      dir: string
+      dir?: string | null
       /** HTML string of processed article content */
       content: string
       /** non-HTML version of `content`  */
@@ -157,14 +157,13 @@ declare module '@omnivore/readability' {
       /** Article description, or short excerpt from the content */
       excerpt: string
       /** Article site name */
-      siteName: string
+      siteName?: string | null
       /** Article site icon */
-      siteIcon: string
+      siteIcon?: string | null
       /** Article preview image */
       previewImage?: string | null
       /** Article published date */
       publishedDate?: Date | null
-      dom?: Element | null
       language?: string | null
     }
   }

packages/api/src/schema.ts

@@ -528,14 +528,14 @@ const schema = gql`
 
   input ParseResult {
     title: String!
-    byline: String!
-    dir: String!
+    byline: String
+    dir: String
     content: String!
     textContent: String!
     length: Int!
     excerpt: String!
-    siteName: String!
-    siteIcon: String!
+    siteName: String
+    siteIcon: String
     previewImage: String
     publishedDate: Date
     language: String

packages/api/src/services/save_email.ts

@@ -41,6 +41,7 @@ export const saveEmail = async (
         // can leave this empty for now
       },
     },
+    null,
     true
   )
   const content = parseResult.parsedContent?.content || input.originalContent
@@ -62,15 +63,21 @@ export const saveEmail = async (
     }),
     pageType: parseResult.pageType,
     hash: stringToHash(content),
-    image: metadata?.previewImage || parseResult.parsedContent?.previewImage,
-    publishedAt: validatedDate(parseResult.parsedContent?.publishedDate),
+    image:
+      metadata?.previewImage ||
+      parseResult.parsedContent?.previewImage ||
+      undefined,
+    publishedAt: validatedDate(
+      parseResult.parsedContent?.publishedDate ?? undefined
+    ),
     createdAt: new Date(),
     savedAt: new Date(),
     readingProgressAnchorIndex: 0,
     readingProgressPercent: 0,
     subscription: input.author,
     state: ArticleSavingRequestStatus.Succeeded,
-    siteIcon: parseResult.parsedContent?.siteIcon,
+    siteIcon: parseResult.parsedContent?.siteIcon ?? undefined,
+    siteName: parseResult.parsedContent?.siteName ?? undefined,
   }
 
   const page = await getPageByParam({

packages/api/src/services/save_page.ts

@@ -226,7 +226,7 @@ export const parsedContentToPage = ({
       croppedPathname ||
       parsedContent?.siteName ||
       url,
-    author: parsedContent?.byline,
+    author: parsedContent?.byline ?? undefined,
     url: normalizeUrl(canonicalUrl || url, {
       stripHash: true,
       stripWWW: false,
@@ -241,9 +241,9 @@ export const parsedContentToPage = ({
     state: ArticleSavingRequestStatus.Succeeded,
     createdAt: saveTime || new Date(),
     savedAt: saveTime || new Date(),
-    siteName: parsedContent?.siteName,
+    siteName: parsedContent?.siteName ?? undefined,
     language: parsedContent?.language ?? undefined,
-    siteIcon: parsedContent?.siteIcon,
+    siteIcon: parsedContent?.siteIcon ?? undefined,
     wordsCount: wordsCount(parsedContent?.textContent || ''),
   }
 }

packages/api/src/utils/parser.ts

@@ -229,8 +229,9 @@ export const parsePreparedContent = async (
     // Format code blocks
     // TODO: we probably want to move this type of thing
     // to the handlers, and have some concept of postHandle
-    if (article?.dom) {
-      const codeBlocks = article.dom.querySelectorAll('code')
+    if (article?.content) {
+      const articleDom = parseHTML(article.content).document
+      const codeBlocks = articleDom.querySelectorAll('code')
       if (codeBlocks.length > 0) {
         codeBlocks.forEach((e) => {
           if (e.textContent) {
@@ -246,12 +247,10 @@ export const parsePreparedContent = async (
             e.replaceWith(code)
           }
         })
-        article.content = article.dom.outerHTML
+        article.content = articleDom.documentElement.outerHTML
       }
 
-      if (article?.dom) {
-        highlightData = findEmbeddedHighlight(article?.dom)
-      }
+      highlightData = findEmbeddedHighlight(articleDom.documentElement)
 
       const ANCHOR_ELEMENTS_BLOCKED_ATTRIBUTES = [
         'omnivore-highlight-id',
@@ -260,7 +259,7 @@ export const parsePreparedContent = async (
       ]
 
       // Get the top level element?
-      const pageNode = article.dom.firstElementChild as HTMLElement
+      const pageNode = articleDom.firstElementChild as HTMLElement
       const nodesToVisitStack: [HTMLElement] = [pageNode]
       const visitedNodeList = []
 
@@ -290,7 +289,7 @@ export const parsePreparedContent = async (
         node.setAttribute('data-omnivore-anchor-idx', (index + 1).toString())
       })
 
-      article.content = article.dom.outerHTML
+      article.content = articleDom.documentElement.outerHTML
     }
 
     const newWindow = parseHTML('')

packages/puppeteer-parse/index.js

@@ -23,7 +23,7 @@ puppeteer.use(StealthPlugin());
 
 // Add adblocker plugin to block all ads and trackers (saves bandwidth)
 const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker');
-puppeteer.use(AdblockerPlugin({ blockTrackers: true }));
+// puppeteer.use(AdblockerPlugin({ blockTrackers: true }));
 
 const storage = new Storage();
 const ALLOWED_ORIGINS = process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : [];
@@ -200,6 +200,35 @@ const sendCreateArticleMutation = async (userId, input) => {
   return response.data.data.createArticle;
 };
 
+const sendSavePageMutation = async (userId, input) => {
+  const data = JSON.stringify({
+    query: `mutation SavePage ($input: SavePageInput!){
+          savePage(input:$input){
+            ... on SaveSuccess{
+              url
+              clientRequestId
+            }
+            ... on SaveError{
+                errorCodes
+            }
+          }
+    }`,
+    variables: {
+      input: Object.assign({}, input , { source: 'puppeteer-parse' }),
+    },
+  });
+
+  const auth = await signToken({ uid: userId }, process.env.JWT_SECRET);
+  const response = await axios.post(`${process.env.REST_BACKEND_ENDPOINT}/graphql`, data,
+    {
+      headers: {
+        Cookie: `auth=${auth};`,
+        'Content-Type': 'application/json',
+      },
+    });
+  return response.data.data.savePage;
+};
+
 const saveUploadedPdf = async (userId, url, uploadFileId, articleSavingRequestId) => {
   return sendCreateArticleMutation(userId, {
       url: encodeURI(url),
@@ -283,18 +312,12 @@ async function fetchContent(req, res) {
 
       const readabilityResult = content ? (await getReadabilityResult(url, content)) : null;
 
-      const apiResponse = await sendCreateArticleMutation(userId, {
+      const apiResponse = await sendSavePageMutation(userId, {
         url: finalUrl,
-        articleSavingRequestId,
-        preparedDocument: {
-          document: content,
-          pageInfo: {
-            title,
-            canonicalUrl: finalUrl,
-          },
-        },
-        skipParsing: !!readabilityResult,
-        readabilityResult,
+        clientRequestId: articleSavingRequestId,
+        title,
+        originalContent: content,
+        parseResult: readabilityResult,
       });
 
       logRecord.totalTime = Date.now() - functionStartTime;
@@ -312,18 +335,12 @@ async function fetchContent(req, res) {
 
     const readabilityResult = content ? (await getReadabilityResult(url, content)) : null;
 
-    const apiResponse = await sendCreateArticleMutation(userId, {
-      url: sbUrl,
-      articleSavingRequestId,
-      preparedDocument: {
-        document: content,
-        pageInfo: {
-          title: sbResult.title,
-          canonicalUrl: sbUrl,
-        },
-      },
-      skipParsing: !!readabilityResult,
-      readabilityResult,
+    const apiResponse = await sendSavePageMutation(userId, {
+      url: finalUrl,
+      clientRequestId: articleSavingRequestId,
+      title,
+      originalContent: content,
+      parseResult: readabilityResult,
     });
 
     logRecord.totalTime = Date.now() - functionStartTime;

packages/readabilityjs/Readability.js

@@ -2998,7 +2998,6 @@ Readability.prototype = {
       siteIcon: metadata.siteIcon,
       previewImage: metadata.previewImage,
       publishedDate: metadata.publishedDate || publishedAt || this._articlePublishedDate,
-      dom: articleContent,
       language: this._getLanguage(metadata.locale || this._languageCode),
     };
   }