From d9c46d00e76c1432d6f7e2c70d6ca760811ef268 Mon Sep 17 00:00:00 2001
From: Christopher Gurnee <chris@gurneeconsulting.net>
Date: Sat, 23 Oct 2021 12:58:06 -0400
Subject: [PATCH] Be more descriptive with removed comments

---
 src/api/pushshift/index.js  |  2 +-
 src/pages/thread/Comment.js | 23 +++++++++++++++++++++--
 src/utils.js                |  9 +++++++++
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/api/pushshift/index.js b/src/api/pushshift/index.js
index df3d9c1..5f836d7 100644
--- a/src/api/pushshift/index.js
+++ b/src/api/pushshift/index.js
@@ -1,6 +1,6 @@
 const chunkSize = 100;
 const postURL    = 'https://api.pushshift.io/reddit/submission/search/?ids='
-const commentURL = `https://api.pushshift.io/reddit/comment/search/?size=${chunkSize}&sort=asc&fields=author,body,created_utc,id,link_id,parent_id,score,subreddit&q=*&link_id=`
+const commentURL = `https://api.pushshift.io/reddit/comment/search/?size=${chunkSize}&sort=asc&fields=author,body,created_utc,id,link_id,parent_id,retrieved_on,retrieved_utc,score,subreddit&q=*&link_id=`
 
 const sleep = ms =>
   new Promise(slept => setTimeout(slept, ms))
diff --git a/src/pages/thread/Comment.js b/src/pages/thread/Comment.js
index ca58a62..25d96f5 100644
--- a/src/pages/thread/Comment.js
+++ b/src/pages/thread/Comment.js
@@ -1,6 +1,6 @@
 import React from 'react'
 import { Link } from 'react-router-dom'
-import { prettyScore, prettyDate, parse, isRemoved } from '../../utils'
+import { prettyScore, prettyDate, prettyTimeDiff, parse, isRemoved } from '../../utils'
 
 const Comment = (props) => {
   let commentStyle = 'comment '
@@ -13,7 +13,26 @@ const Comment = (props) => {
     commentStyle += props.depth % 2 === 0 ? 'comment-even' : 'comment-odd'
   }
 
-  const innerHTML = (isRemoved(props.body) && props.removed) ? '<p>[removed too quickly to be archived]</p>' : parse(props.body)
+  let innerHTML;
+  if (isRemoved(props.body) && props.removed) {
+    if (!props.hasOwnProperty('retrieved_utc') && !props.hasOwnProperty('retrieved_on') || !props.hasOwnProperty('created_utc')) {
+      innerHTML = '<p>[removed too quickly to be archived]</p>'
+    } else if (props.created_utc < 1627776000) {  // Aug 1 2021
+      const retrieved = props.hasOwnProperty('retrieved_utc') ? props.retrieved_utc : props.retrieved_on;
+      innerHTML = '<p>[removed within ${prettyTimeDiff(retrieved - props.created_utc)}]</p>'
+    }
+    // After around Aug 1 2021, Pushshift began updating comments from Reddit after around
+    // 24-48 hours, including removing(?) comments that were removed from Reddit. The presence
+    // of either retrieved_utc or retrieved_on can currently be used to test for this behaviour.
+    else if (props.hasOwnProperty('retrieved_utc')) {
+      innerHTML = `<p>[removed within ${prettyTimeDiff(props.retrieved_utc - props.created_utc)}]</p>`
+    } else {
+      innerHTML = `<p>[either removed too quickly, or <a href='https://www.reddit.com/r/pushshift/comments/pgzdav/the_api_now_appears_to_rewrite_nearly_all/'>removed(?) from archive</a> after ${prettyTimeDiff(props.retrieved_on - props.created_utc)}]</p>`
+    }
+  } else {
+    innerHTML = parse(props.body)
+  }
+
   const permalink = `/r/${props.subreddit}/comments/${props.link_id}/_/${props.id}/`
 
   return (
diff --git a/src/utils.js b/src/utils.js
index f65333c..cb4ba51 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -50,6 +50,15 @@ export const prettyDate = createdUTC => {
   return `${Math.floor(dayDiff / 365)} years ago`
 }
 
+// Time difference in seconds to text, rounded up (e.g. x seconds/minutes/hours)
+export const prettyTimeDiff = secondDiff => {
+  if (secondDiff < 2) return `1 second`
+  if (secondDiff < 120) return `${secondDiff} seconds`
+  if (secondDiff < 7200) return `${Math.ceil(secondDiff / 60)} minutes`
+  if (secondDiff < 172800) return `${Math.ceil(secondDiff / 3600)} hours`
+  return `${Math.ceil(secondDiff / 86400)} days`
+}
+
 // Reddit format for scores, e.g. 12000 => 12k
 export const prettyScore = score => {
   if (score >= 100000) {