From d69e732579fb3c6a1ab194df72fdc1ff8ca72368 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Wed, 24 May 2023 20:29:35 +0800
Subject: [PATCH] fix: dedupe url if failed to save

---
 packages/api/src/utils/parser.ts              | 21 ++++++++++++++-----
 .../src/websites/twitter-handler.ts           |  3 ++-
 packages/puppeteer-parse/index.js             | 18 +++++++---------
 3 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts
index 6d255677a..e48ca7326 100644
--- a/packages/api/src/utils/parser.ts
+++ b/packages/api/src/utils/parser.ts
@@ -207,6 +207,16 @@ export const parsePreparedContent = async (
   let highlightData = undefined
   const { document, pageInfo } = preparedDocument
 
+  if (!document) {
+    console.log('No document')
+    return {
+      canonicalUrl: url,
+      parsedContent: null,
+      domContent: '',
+      pageType: PageType.Unknown,
+    }
+  }
+
   // Checking for content type acceptance or if there are no contentType
   // at all (backward extension versions compatibility)
   if (
@@ -222,14 +232,15 @@ export const parsePreparedContent = async (
     }
   }
 
-  let dom = parseHTML(document).document
+  let dom: Document | null = null
 
   try {
+    dom = parseHTML(document).document
+
     if (!article) {
       // Attempt to parse the article
       // preParse content
-      const preParsedDom = await preParseContent(url, dom)
-      preParsedDom && (dom = preParsedDom)
+      dom = (await preParseContent(url, dom)) || dom
 
       article = await getReadabilityResult(url, document, dom, isNewsletter)
     }
@@ -260,7 +271,7 @@ export const parsePreparedContent = async (
         codeBlocks.forEach((e) => {
           if (e.textContent) {
             const att = hljs.highlightAuto(e.textContent)
-            const code = dom.createElement('code')
+            const code = articleDom.createElement('code')
             const langClass =
               `hljs language-${att.language}` +
               (att.second_best?.language
@@ -356,7 +367,7 @@ export const parsePreparedContent = async (
     domContent: document,
     parsedContent: article,
     canonicalUrl,
-    pageType: parseOriginalContent(dom),
+    pageType: dom ? parseOriginalContent(dom) : PageType.Unknown,
     highlightData,
   }
 }
diff --git a/packages/content-handler/src/websites/twitter-handler.ts b/packages/content-handler/src/websites/twitter-handler.ts
index 832cc7e13..c1878e498 100644
--- a/packages/content-handler/src/websites/twitter-handler.ts
+++ b/packages/content-handler/src/websites/twitter-handler.ts
@@ -288,7 +288,8 @@ const getTweetIds = async (
       return Array.from(ids)
     }, author)) as string[]
   } catch (error) {
-    console.log(error)
+    console.error('Error getting tweets', error)
+
     return []
   } finally {
     if (context) {
diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js
index f4e5e55c9..352137ac8 100644
--- a/packages/puppeteer-parse/index.js
+++ b/packages/puppeteer-parse/index.js
@@ -73,8 +73,9 @@ const fetchContentWithScrapingBee = async (url) => {
     const dom = parseHTML(response.data).document;
     return { title: dom.title, domContent: dom.documentElement.outerHTML, url }
   } catch (e) {
-    console.log('error fetching with scrapingbee', e)
-    return { title: '', domContent: '', url }
+    console.error('error fetching with scrapingbee', e.message)
+
+    return { title: url, domContent: '', url }
   }
 }
 
@@ -325,19 +326,16 @@ async function fetchContent(req, res) {
 
       let readabilityResult = null;
       if (content) {
-        let document = parseHTML(content).document;
+        const document = parseHTML(content).document;
 
         // preParse content
-        const preParsedDom = await preParseContent(url, document)
-        if (preParsedDom) {
-          document = preParsedDom
-        }
+        const preParsedDom = (await preParseContent(url, document)) || document;
 
-        readabilityResult = await getReadabilityResult(url, document);
+        readabilityResult = await getReadabilityResult(url, preParsedDom);
       }
 
       const apiResponse = await sendSavePageMutation(userId, {
-        url: finalUrl,
+        url,
         clientRequestId: articleSavingRequestId,
         title,
         originalContent: content,
@@ -373,7 +371,7 @@ async function fetchContent(req, res) {
     }
 
     const apiResponse = await sendSavePageMutation(userId, {
-      url: finalUrl,
+      url,
       clientRequestId: articleSavingRequestId,
       title,
       originalContent: content,