From 20eae33207f8dba89918cfee40e964f5578e9dbb Mon Sep 17 00:00:00 2001
From: Christopher Gurnee <chris@gurneeconsulting.net>
Date: Sat, 30 Oct 2021 12:51:04 -0400
Subject: [PATCH] Use Reddit for comments with missing parents

Pushshift occasionally fails to return comments which are the parents
of comments that it does return. Look these parents up in the Reddit
API and use its results instead, otherwise entire comment trees may be
hidden. They could still be hidden if two consecutive parents are
missing from Pushshift, but hopefully that isn't a common occurance.
---
 src/pages/thread/index.js | 45 ++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/src/pages/thread/index.js b/src/pages/thread/index.js
index 2b77268..1ea3cc7 100644
--- a/src/pages/thread/index.js
+++ b/src/pages/thread/index.js
@@ -51,31 +51,39 @@ class Thread extends React.Component {
     // Get comment ids from pushshift
     getPushshiftComments(threadID)
       .then(pushshiftComments => {
+        console.log(`Pushshift: ${pushshiftComments.length} comments`)
+        const pushshiftCommentLookup = new Map(pushshiftComments.map(c => [c.id, c]))
+        const ids = []
+
         // Extract ids from pushshift response
-        const ids = pushshiftComments.map(comment => comment.id)
-        this.props.global.setLoading('Comparing comments to Reddit API...')
+        pushshiftComments.forEach(comment => {
+          ids.push(comment.id)
+          if (comment.parent_id != threadID && !pushshiftCommentLookup.has(comment.parent_id)) {
+            ids.push(comment.parent_id)
+          }
+        });
+
         // Get all the comments from reddit
+        this.props.global.setLoading('Comparing comments to Reddit API...')
         return getRedditComments(ids)
           .then(redditComments => {
-            // Temporary lookup for updating score
-            const redditCommentLookup = {}
-            redditComments.forEach(comment => {
-              redditCommentLookup[comment.id] = comment
-            })
-
-            // Replace pushshift score with reddit (its usually more accurate)
-            pushshiftComments.forEach(comment => {
-              const redditComment = redditCommentLookup[comment.id]
-              if (redditComment !== undefined) {
-                comment.score = redditComment.score
-              }
-            })
-
+            console.log(`Reddit: ${redditComments.length} comments`)
             const removed = []
             const deleted = []
 
-            // Check what as removed / deleted according to reddit
             redditComments.forEach(comment => {
+              const pushshiftComment = pushshiftCommentLookup.get(comment.id)
+              if (pushshiftComment === undefined) {
+                // When a parent comment is missing from pushshift, use the reddit comment instead
+                comment.parent_id = comment.parent_id.substring(3)
+                comment.link_id = comment.link_id.substring(3)
+                pushshiftComments.push(comment)
+              } else {
+                // Replace pushshift score with reddit (it's usually more accurate)
+                pushshiftComment.score = comment.score
+              }
+
+              // Check what is removed / deleted according to reddit
               if (isRemoved(comment.body)) {
                 removed.push(comment.id)
               } else if (isDeleted(comment.body)) {
@@ -83,9 +91,6 @@ class Thread extends React.Component {
               }
             })
 
-            console.log(`Pushshift: ${pushshiftComments.length} comments`)
-            console.log(`Reddit: ${redditComments.length} comments`)
-
             this.props.global.setSuccess()
             this.setState({
               pushshiftComments,