From 0abf9222fbbf2627d3a4ce3599d3c23156d02cf5 Mon Sep 17 00:00:00 2001 From: Christopher Gurnee Date: Fri, 18 Mar 2022 18:10:20 +0000 Subject: [PATCH] Improve permalinks and add parent link to comments Beforehand, visiting a permalink would cause Unddit to begin loading from the Post's first comment. If the max-to-download setting was too small, and the permalinked comment wasn't yet encountered, no comment tree would be displayed. Now, only comments that occur at the same time as the permalinked comment or later (chronologically) are downloaded. This also adds a Parent link to each comment. When viewing a permalinked comment, clicking the top-level Parent link will download only new comments between the parent and child comments. Likewise, clicking the 'view rest of comments' link will only download new comments from the first comment up until the previously viewed comment. In general, Unddit will not re-download comments that it knows it has previously downloaded, nor download more than max-to-download comments per link clicked. This can leave 'gaps' in the list of downloaded comments. If the 'load more comments' link is clicked, Unddit will only download comments that can fill such gaps between the currently-linked permalink (or if not viewing one, the first comment) and the last, up to the max-to-download setting. --- dist/main.css | 2 +- src/api/pushshift/index.js | 4 +- src/pages/common/Post.js | 4 +- src/pages/thread/Comment.js | 15 +- src/pages/thread/CommentSection.js | 64 ++++--- src/pages/thread/SortBy.js | 4 +- src/pages/thread/index.js | 275 ++++++++++++++++++++++++----- src/sass/comment.sass | 6 + src/sass/thread.sass | 5 +- 9 files changed, 304 insertions(+), 75 deletions(-) diff --git a/dist/main.css b/dist/main.css index b2e5500..b875672 100644 --- a/dist/main.css +++ b/dist/main.css @@ -1 +1 @@ -html,body{margin:0;padding:0;background-color:#262626;font-family:verdana,arial,helvetica,sans-serif}.wait{cursor:wait}.main{margin:15px;color:#fff}.removed{background-color:#840c09}.deleted{background-color:#00007d}.removed-text{color:#c70300}.deleted-text{color:blue}a{color:#8cb3d9}a:link,a:visited,a:active{text-decoration:none}a:hover{text-decoration:underline}.author{color:#6a98af}.author:not([href]){color:inherit;text-decoration:none}.nowrap{display:inline-block}.space{margin-left:5px}select{background:transparent;border:0;color:#828282}header{background-color:#c70300;display:flex;justify-content:space-between;align-items:center;padding:10px}header #header a{color:#fff}header #header h1{margin:7px 0;text-align:center}header #header h1 a{transition:color .3s}header #header h1 a:hover{text-decoration:none;color:#ff8b88}header #header nav a{margin:0 7px;font-size:20px}header #status{display:flex;justify-content:space-between;align-items:center}header #status #status-text,header #status #status-helpurl{color:#fff;margin:0 20px 0 0}header #status #status-helpurl{font-size:20px}header #status #status-image{height:64px;width:64px;border-radius:32px}#main-box{margin:0 auto;padding:20px;border-radius:5px;color:#ddd;word-wrap:break-word;max-width:800px;background-color:#161616}#main-box h2{margin:20px 0px 12px}#main-box h2.about{margin:0;color:#c70300}#main-box h2.todo{color:#239f2b}#main-box h2.contact{color:#1b767a}#main-box .highlighted{animation:highlighted-anim 3s ease-out}@keyframes highlighted-anim{0%{background-color:#add8e6}}#main-box .question{font-size:18px;color:#add8e6}#main-box ul{padding-left:30px;margin:0}#main-box .bookmarklet{background:#c70300;color:#fff;padding:9px;font-size:large;font-weight:bold;display:inline-block;border-radius:5px;margin:0 7px}.comment{min-width:240px;margin:0 0 8px 0;padding:5px 8px 5px 20px;border:1px solid #333;border-radius:3px;color:#ddd;font-size:14px}.comment .comment-head{font-size:10px}.comment .comment-collapse{cursor:pointer}.comment .comment-collapsed span,.comment .comment-collapsed a:not(.comment-collapse){color:#828282;font-style:oblique}.comment .comment-author{font-weight:bold}.comment .comment-author:not([href]){color:#828282}.comment .comment-poster{color:#ddd;background-color:#0055df;padding:0 2px;border-radius:3px}.comment .comment-score{color:#b4b4b4;font-weight:bold}.comment .comment-time{color:#828282}.comment .comment-body{max-width:840px;line-height:20px}.comment .comment-body p{margin:5px 0;line-height:20px}.comment .comment-body a:hover{text-decoration:none}.comment .comment-links{margin-bottom:6px}.comment .comment-links a{color:#828282;font-weight:bold;font-size:10px;margin-right:4px;cursor:pointer}.comment-odd{background-color:#121212}.comment-even{background-color:#161616}#comment-info{margin-top:10px;font-weight:bold;padding-bottom:5px;border-bottom:1px dotted gray;font-size:16px}#comment-info span{margin-right:8px}#comment-sort,#comment-sort input{color:gray;font-size:12px;margin:5px 0;background-color:#262626}#comment-sort input{border-color:#767676}#comment-sort input[type=number]{width:54px;appearance:textfield}#comment-sort input[type=number]:hover,#comment-sort input[type=number]:focus{appearance:none}#comment-sort input:hover,#comment-sort input:focus,#comment-sort select:hover,#comment-sort select:focus{color:#ddd}#comment-sort option{color:#ddd;background-color:#262626}.thread{display:flex;flex-grow:1;margin-bottom:7px;border-radius:1px;padding-top:6px}.thread .thumbnail{width:70px;margin-right:3px;margin-top:-2px}.thread .thumbnail img{border-radius:1px}.thread .thumbnail-default{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1099px;background-repeat:no-repeat;height:50px}.thread .thumbnail-self{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1267px;background-repeat:no-repeat;height:50px}.thread .thumbnail-image{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1043px;background-repeat:no-repeat;height:50px}.thread .thumbnail-nsfw{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1155px;background-repeat:no-repeat;height:50px}.thread .thread-score-box{margin:0 7px;width:43px}.thread .vote{background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAcCAMAAADC4sagAAABFFBMVEUjJCQiIiIjIyMkIiIkJCQlJSUlJicmJiYnIyEnJycnKCsoKCgpLC8rIyErKyssMDUvLy8wJCAwMDAxMTEyMjI0NDQ2NjY3JR83Nzc5OTk7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NGRkZISEhKSkpKV2tLS0tMTExOTk5PT09QYHdSUlJTU1NVVVVWVlZXV1dXaINYWFhaWlpabYlbW1tecpBgYGBhYWFid5dkZGRle5xlfJ1pgKRrhKhshKlshapuLhZwirFyjbV0j7h4lL97mcZ9MBR/nsyCo9OOMxGcNQ+eNg+kNw6sOA2vOAy0OQy7Ogq/OwrBOwrDPAnLPQjPPQfTPgfbPwXjQQTrQgPzQwLKxGgxAAAAAXRSTlMAQObYZgAAAAlwSFlzAAALEwAACxMBAJqcGAAAAVlJREFUOMu9lF1TwjAQRTcMvmeQOi1QPmxFBQTxW1ERUUFEUVRU/P//w9yGmBTFyZPnpXNmbju76W6IJM8Uw49rLa6Tz4mpq/mCqY3Gvqnj6XQ61lp0mGd8favCwx2tow8wUlpwGWOZrNJKjXO+UVc6fJcMZzV7IswSOU/qej0l0rwyq33wphhAM1kWkcy70GB7mUfUNqG9V02PyPMTMs2Wig5RaTctwzxVXyPqvJh0VkJBIKJ4huncCRAvHOEZyGKegO4a6UDrIdLGCT4C6/QDsE7fA+v0HdAadWqk0SH9D3sgXkm4uJIDYKTFnwwXd3kMrNOnwDp9BmzS5QuTshN1iDRwfkxV9VJTJXLzSWYM+NzEEjWvFE2oGvCE7/2yDeL8riWzM8xmorRazLlNI2rdgNb3ZYLF1Es/t8VE7X6/39Yqlt4tLLwhiLq3XVNLfumP24fo3Opm+wLGCVa252Y8tQAAAABJRU5ErkJggg==);background-repeat:no-repeat;margin-left:auto;margin-right:auto;width:15px;height:14px;cursor:pointer}.thread .upvote{background-position:-15px 0px;margin-top:2px}.thread .downvote{background-position:-15px -14px;margin-top:2px}.thread .thread-score{color:#646464;font-size:13px;font-weight:bold;text-align:center}.thread .thread-content{flex:1;float:left;margin-left:3px}.thread .link-flair{display:inline-block;color:#c8c8c8;background-color:#404040;margin-right:5px;border:1px solid #4d4d4d;border-radius:2px;padding:0 2px;font-size:10px}.thread .thread-title{color:#a6a6a6;margin-right:5px}.thread .thread-title:hover{text-decoration:none}.thread .domain{color:#888;font-size:10px}.thread .thread-selftext{border:1px solid #666;border-radius:7px;margin:5px 0 7px;padding:5px 10px;max-width:840px;color:#ddd;font-size:14px}.thread .thread-selftext p{margin:5px 0;line-height:20px}.thread .thread-selftext a:hover{text-decoration:none}.thread .thread-selftext p:first-child{margin-top:0px}.thread .thread-selftext p:last-child{margin-bottom:0px}.thread .thread-image{max-width:768px;max-height:768px}.thread .thread-info{color:#828282;font-size:10px;margin-top:2px}.thread .total-comments{font-size:10px;font-weight:bold;margin:4px 0}.thread .total-comments a,.thread .total-comments span{color:#828282;margin-right:4px}.thread .total-comments a{cursor:pointer}.view-rest-of-comment{background-color:#264d73;border-color:#2966a3;font-size:13px;padding:10px;color:#ccc;margin-bottom:10px}.load-more{font-weight:bold;font-size:10px}.load-more a,.load-more span{margin-left:16px}.load-more a{cursor:pointer}.load-more .fade{animation:fade-anim 3s 2s ease-in-out forwards}@keyframes fade-anim{100%{color:#262626}}.post-rank{min-width:20px;color:#505050;font-size:16px;text-align:right;margin:14px 4px 0 0} +html,body{margin:0;padding:0;background-color:#262626;font-family:verdana,arial,helvetica,sans-serif}.wait{cursor:wait}.main{margin:15px;color:#fff}.removed{background-color:#840c09}.deleted{background-color:#00007d}.removed-text{color:#c70300}.deleted-text{color:blue}a{color:#8cb3d9}a:link,a:visited,a:active{text-decoration:none}a:hover{text-decoration:underline}.author{color:#6a98af}.author:not([href]){color:inherit;text-decoration:none}.nowrap{display:inline-block}.space{margin-left:5px}select{background:transparent;border:0;color:#828282}header{background-color:#c70300;display:flex;justify-content:space-between;align-items:center;padding:10px}header #header a{color:#fff}header #header h1{margin:7px 0;text-align:center}header #header h1 a{transition:color .3s}header #header h1 a:hover{text-decoration:none;color:#ff8b88}header #header nav a{margin:0 7px;font-size:20px}header #status{display:flex;justify-content:space-between;align-items:center}header #status #status-text,header #status #status-helpurl{color:#fff;margin:0 20px 0 0}header #status #status-helpurl{font-size:20px}header #status #status-image{height:64px;width:64px;border-radius:32px}#main-box{margin:0 auto;padding:20px;border-radius:5px;color:#ddd;word-wrap:break-word;max-width:800px;background-color:#161616}#main-box h2{margin:20px 0px 12px}#main-box h2.about{margin:0;color:#c70300}#main-box h2.todo{color:#239f2b}#main-box h2.contact{color:#1b767a}#main-box .highlighted{animation:highlighted-anim 3s ease-out}@keyframes highlighted-anim{0%{background-color:#add8e6}}#main-box .question{font-size:18px;color:#add8e6}#main-box ul{padding-left:30px;margin:0}#main-box .bookmarklet{background:#c70300;color:#fff;padding:9px;font-size:large;font-weight:bold;display:inline-block;border-radius:5px;margin:0 7px}.comment{min-width:240px;margin:0 0 8px 0;padding:5px 8px 5px 20px;border:1px solid #333;border-radius:3px;color:#ddd;font-size:14px}.comment .comment-head{font-size:10px}.comment .comment-collapse{cursor:pointer}.comment .comment-collapsed span,.comment .comment-collapsed a:not(.comment-collapse){color:#828282;font-style:oblique}.comment .comment-author{font-weight:bold}.comment .comment-author:not([href]){color:#828282}.comment .comment-poster{color:#ddd;background-color:#0055df;padding:0 2px;border-radius:3px}.comment .comment-score{color:#b4b4b4;font-weight:bold}.comment .comment-time{color:#828282}.comment .comment-body{max-width:840px;line-height:20px}.comment .comment-body p{margin:5px 0;line-height:20px}.comment .comment-body a:hover{text-decoration:none}.comment .comment-links{margin-bottom:6px}.comment .comment-links a{color:#828282;font-weight:bold;font-size:10px;margin-right:4px;cursor:pointer}.comment .comment-links a.wait{cursor:wait}.comment .comment-links a.wait:hover{text-decoration:none}.comment-odd{background-color:#121212}.comment-even{background-color:#161616}#comment-info{margin-top:10px;font-weight:bold;padding-bottom:5px;border-bottom:1px dotted gray;font-size:16px}#comment-info span{margin-right:8px}#comment-sort,#comment-sort input{color:gray;font-size:12px;margin:5px 0;background-color:#262626}#comment-sort input{border-color:#767676}#comment-sort input[type=number]{width:54px;appearance:textfield}#comment-sort input[type=number]:hover,#comment-sort input[type=number]:focus{appearance:none}#comment-sort input:hover,#comment-sort input:focus,#comment-sort select:hover,#comment-sort select:focus{color:#ddd}#comment-sort option{color:#ddd;background-color:#262626}.thread{display:flex;flex-grow:1;margin-bottom:7px;border-radius:1px;padding-top:6px}.thread .thumbnail{width:70px;margin-right:3px;margin-top:-2px}.thread .thumbnail img{border-radius:1px}.thread .thumbnail-default{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1099px;background-repeat:no-repeat;height:50px}.thread .thumbnail-self{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1267px;background-repeat:no-repeat;height:50px}.thread .thumbnail-image{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1043px;background-repeat:no-repeat;height:50px}.thread .thumbnail-nsfw{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1155px;background-repeat:no-repeat;height:50px}.thread .thread-score-box{margin:0 7px;width:43px}.thread .vote{background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAcCAMAAADC4sagAAABFFBMVEUjJCQiIiIjIyMkIiIkJCQlJSUlJicmJiYnIyEnJycnKCsoKCgpLC8rIyErKyssMDUvLy8wJCAwMDAxMTEyMjI0NDQ2NjY3JR83Nzc5OTk7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NGRkZISEhKSkpKV2tLS0tMTExOTk5PT09QYHdSUlJTU1NVVVVWVlZXV1dXaINYWFhaWlpabYlbW1tecpBgYGBhYWFid5dkZGRle5xlfJ1pgKRrhKhshKlshapuLhZwirFyjbV0j7h4lL97mcZ9MBR/nsyCo9OOMxGcNQ+eNg+kNw6sOA2vOAy0OQy7Ogq/OwrBOwrDPAnLPQjPPQfTPgfbPwXjQQTrQgPzQwLKxGgxAAAAAXRSTlMAQObYZgAAAAlwSFlzAAALEwAACxMBAJqcGAAAAVlJREFUOMu9lF1TwjAQRTcMvmeQOi1QPmxFBQTxW1ERUUFEUVRU/P//w9yGmBTFyZPnpXNmbju76W6IJM8Uw49rLa6Tz4mpq/mCqY3Gvqnj6XQ61lp0mGd8favCwx2tow8wUlpwGWOZrNJKjXO+UVc6fJcMZzV7IswSOU/qej0l0rwyq33wphhAM1kWkcy70GB7mUfUNqG9V02PyPMTMs2Wig5RaTctwzxVXyPqvJh0VkJBIKJ4huncCRAvHOEZyGKegO4a6UDrIdLGCT4C6/QDsE7fA+v0HdAadWqk0SH9D3sgXkm4uJIDYKTFnwwXd3kMrNOnwDp9BmzS5QuTshN1iDRwfkxV9VJTJXLzSWYM+NzEEjWvFE2oGvCE7/2yDeL8riWzM8xmorRazLlNI2rdgNb3ZYLF1Es/t8VE7X6/39Yqlt4tLLwhiLq3XVNLfumP24fo3Opm+wLGCVa252Y8tQAAAABJRU5ErkJggg==);background-repeat:no-repeat;margin-left:auto;margin-right:auto;width:15px;height:14px;cursor:pointer}.thread .upvote{background-position:-15px 0px;margin-top:2px}.thread .downvote{background-position:-15px -14px;margin-top:2px}.thread .thread-score{color:#646464;font-size:13px;font-weight:bold;text-align:center}.thread .thread-content{flex:1;float:left;margin-left:3px}.thread .link-flair{display:inline-block;color:#c8c8c8;background-color:#404040;margin-right:5px;border:1px solid #4d4d4d;border-radius:2px;padding:0 2px;font-size:10px}.thread .thread-title{color:#a6a6a6;margin-right:5px}.thread .thread-title:hover{text-decoration:none}.thread .domain{color:#888;font-size:10px}.thread .thread-selftext{border:1px solid #666;border-radius:7px;margin:5px 0 7px;padding:5px 10px;max-width:840px;color:#ddd;font-size:14px}.thread .thread-selftext p{margin:5px 0;line-height:20px}.thread .thread-selftext a:hover{text-decoration:none}.thread .thread-selftext p:first-child{margin-top:0px}.thread .thread-selftext p:last-child{margin-bottom:0px}.thread .thread-image{max-width:768px;max-height:768px}.thread .thread-info{color:#828282;font-size:10px;margin-top:2px}.thread .total-comments{font-size:10px;font-weight:bold;margin:4px 0}.thread .total-comments a,.thread .total-comments span{color:#828282;margin-right:4px}.thread .total-comments a{cursor:pointer}.view-rest-of-comment{background-color:#264d73;border-color:#2966a3;font-size:13px;padding:10px;color:#ccc;margin-bottom:10px}.view-rest-of-comment .faux-link{color:#8cb3d9}.load-more{font-weight:bold;font-size:10px}.load-more a,.load-more span{margin-left:16px}.load-more a{cursor:pointer}.load-more .fade{animation:fade-anim 3s 5s ease-in-out forwards}@keyframes fade-anim{100%{color:#262626}}.post-rank{min-width:20px;color:#505050;font-size:16px;text-align:right;margin:14px 4px 0 0} diff --git a/src/api/pushshift/index.js b/src/api/pushshift/index.js index 66d55e3..4585a32 100644 --- a/src/api/pushshift/index.js +++ b/src/api/pushshift/index.js @@ -74,7 +74,7 @@ export const getPost = async threadID => { // The callback() function is called with an Array of comments after each chunk is // retrieved. It should return as quickly as possible (scheduling time-taking work // later), and may return false to cause getComments to exit early, or true otherwise. -export const getComments = async (callback, threadID, maxComments, after) => { +export const getComments = async (callback, threadID, maxComments, after = 0, before = undefined) => { let chunks = Math.floor(maxComments / chunkSize), response, lastCreatedUtc = 1 while (true) { @@ -82,7 +82,7 @@ export const getComments = async (callback, threadID, maxComments, after) => { while (true) { await pushshiftTokenBucket.waitForToken() try { - response = await fetchJson(`${commentURL}${threadID}${after ? `&after=${after}` : ''}`) + response = await fetchJson(`${commentURL}${threadID}${after ? `&after=${after}` : ''}${before ? `&before=${before}` : ``}`) break } catch (error) { if (delay >= 8000) // after ~16s of consecutive failures diff --git a/src/pages/common/Post.js b/src/pages/common/Post.js index 3d98971..3a2cd0c 100644 --- a/src/pages/common/Post.js +++ b/src/pages/common/Post.js @@ -86,7 +86,9 @@ export default (props) => { {innerHTML !== undefined &&
}
- {props.num_comments} comments  + {props.reloadingComments ? + {props.num_comments} comments : + {props.num_comments} comments}  reddit  reveddit {props.hasOwnProperty('edited_selftext') && diff --git a/src/pages/thread/Comment.js b/src/pages/thread/Comment.js index 351b89c..ada629f 100644 --- a/src/pages/thread/Comment.js +++ b/src/pages/thread/Comment.js @@ -1,5 +1,5 @@ import React, { useState } from 'react' -import { Link } from 'react-router-dom' +import { Link, NavLink } from 'react-router-dom' import { prettyScore, prettyDate, prettyTimeDiff, exactDateTime, parse, isRemoved } from '../../utils' const Comment = (props) => { @@ -38,6 +38,15 @@ const Comment = (props) => { const [collapsed, setCollapsed] = useState(false) const [showEdited, setShowEdited] = useState(false) const permalink = `/r/${props.subreddit}/comments/${props.link_id}/_/${props.id}/` + const parentlink = props.parent_id == props.link_id ? undefined : ( + props.depth == 0 ? + parent + : + parent + ) return (
@@ -57,7 +66,8 @@ const Comment = (props) => { {prettyScore(props.score)} point{(props.score !== 1) && 's'} - {prettyDate(props.created_utc)} + {props.created_utc && + {prettyDate(props.created_utc)}} {(props.hasOwnProperty('edited_body') || props.edited) && * (last edited {prettyDate(props.edited ? props.edited : props.created_utc)})} @@ -68,6 +78,7 @@ const Comment = (props) => { permalink reddit reveddit + {parentlink} {props.hasOwnProperty('edited_body') && setShowEdited(!showEdited)} onKeyDown={e => e.key == "Enter" && setShowEdited(!showEdited)} diff --git a/src/pages/thread/CommentSection.js b/src/pages/thread/CommentSection.js index c91c201..8f49db7 100644 --- a/src/pages/thread/CommentSection.js +++ b/src/pages/thread/CommentSection.js @@ -6,7 +6,7 @@ import { showRemovedAndDeleted, showRemoved, showDeleted } from '../../utils' -const unflatten = (commentMap, root) => { +const unflatten = (commentMap, rootID, postID) => { const commentTree = [] commentMap.forEach(comment => { @@ -14,28 +14,52 @@ const unflatten = (commentMap, root) => { comment.replies = [] }) - commentMap.forEach(comment => { - if (!comment) - return - const parentID = comment.parent_id - let parentComment + if (rootID == postID) { + commentMap.forEach(comment => { + if (!comment) + return + const parentID = comment.parent_id + if (parentID == postID) + commentTree.push(comment) + else { + const parentComment = commentMap.get(comment.parent_id) + if (parentComment) + parentComment.replies.push(comment) + else + console.warn('Missing parent ID:', parentID, 'for comment', comment) + } + }) + return commentTree - if (parentID === root) { - commentTree.push(comment) - } else if ((parentComment = commentMap.get(parentID)) !== undefined) { - parentComment.replies.push(comment) - } else { - console.warn('Missing parent ID:', parentID, 'for comment', comment) + } else { + const missingRootReplies = [] + commentMap.forEach(comment => { + if (!comment) + return + const parentID = comment.parent_id + const parentComment = commentMap.get(parentID) + if (parentComment) + parentComment.replies.push(comment) + else if (parentID == rootID) + missingRootReplies.push(comment) + }) + let rootComment = commentMap.get(rootID) + if (!rootComment) { + const anyComment = commentMap.values().next().value + if (!anyComment) + return [] + rootComment = { + id: rootID, + link_id: anyComment.link_id, + parent_id: anyComment.link_id, + subreddit: anyComment.subreddit, + score: '?', + body: '...', + replies: missingRootReplies + } } - }) - - let rootComment - if ((rootComment = commentMap.get(root)) !== undefined) { - rootComment.replies = commentTree return [rootComment] } - - return commentTree } const sortCommentTree = (comments, sortFunction) => { @@ -86,7 +110,7 @@ const commentSection = (props) => { ) )) if (needsRebuild) - commentTree = unflatten(props.comments, root) + commentTree = unflatten(props.comments, root, props.postID) if (needsRebuild || commentFilter !== lastFilter) { if (commentFilter === filter.removedDeleted) { diff --git a/src/pages/thread/SortBy.js b/src/pages/thread/SortBy.js index 9c70486..66fe960 100644 --- a/src/pages/thread/SortBy.js +++ b/src/pages/thread/SortBy.js @@ -40,14 +40,14 @@ const sortBy = props => { e.key == "Enter" && e.target.blur()} - onChange= {e => setMaxCommentsField(parseInt(e.target.value))} + onChange= {e => setMaxCommentsField(constrainMaxComments(parseInt(e.target.value)))} onBlur= {e => e.target.value = props.global.setMaxComments(e.target.value)} { ...(isFirefox ? { onClick: e => e.target.focus() } : {}) } defaultValue={props.global.maxComments} type='number' maxLength='5' required min={minCommentsLimit} max={maxCommentsLimit} step={minCommentsLimit} /> - { !props.loadedAllComments && !props.reloadingComments && constrainMaxComments(maxCommentsField) - minCommentsLimit >= props.total && + { !props.reloadingComments && !props.loadedAllComments && maxCommentsField > props.global.maxComments && maxCommentsField - minCommentsLimit >= props.total && props.global.loadMoreComments(props.global.maxComments - props.total)} type='button' value='Reload' /> diff --git a/src/pages/thread/index.js b/src/pages/thread/index.js index 1e48768..02ec736 100644 --- a/src/pages/thread/index.js +++ b/src/pages/thread/index.js @@ -7,7 +7,8 @@ import { } from '../../api/reddit' import { getPost as getPushshiftPost, - getComments as getPushshiftComments + getComments as getPushshiftComments, + chunkSize as pushshiftChunkSize } from '../../api/pushshift' import { isDeleted, isRemoved, sleep } from '../../utils' import { connect, constrainMaxComments } from '../../state' @@ -47,6 +48,9 @@ class ChunkedQueue { } } +// The .firstCreated of the contig containing a post's first comment (see contigs below) +const EARLIEST_CREATED = 1 + class Thread extends React.Component { state = { post: {}, @@ -58,8 +62,33 @@ class Thread extends React.Component { reloadingComments: false } + // A 'contig' is an object representing a contiguous block of comments currently being downloaded or already + // downloaded, e.g. { firstCreated: #, lastCreated: # } (secs past the epoch; min. value of EARLIEST_CREATED) + contigs = [] // sorted non-overlapping array of contig objects + curContigIdx = 0 + curContig () { return this.contigs[this.curContigIdx] } + nextContig () { return this.contigs[this.curContigIdx + 1] } + + // If the current contig and the next probably overlap, merge them + // (should only be called if there's another reason to believe they overlap) + mergeContigs () { + const nextContig = this.nextContig() + if (this.curContig().lastCreated >= nextContig?.firstCreated) // probably; definitely would be '>' + nextContig.firstCreated = this.contigs.splice(this.curContigIdx, 1)[0].firstCreated + else + console.warn("Can't merge contigs", this.curContig(), "and", nextContig) // shouldn't happen + } + + redditIdsToPushshift (comment) { + comment.parent_id = comment.parent_id?.substring(3) || this.props.match.params.threadID + comment.link_id = comment.link_id?.substring(3) || this.props.match.params.threadID + return comment + } + + commentIdAttempts = new Set() // keeps track of attempts to load permalinks to avoid reattempts + componentDidMount () { - const { subreddit, threadID } = this.props.match.params + const { subreddit, threadID, commentID } = this.props.match.params this.props.global.setLoading('Loading post...') // Get post from Reddit. Each code path below should end in either @@ -159,29 +188,140 @@ class Thread extends React.Component { } }) - const maxCommentsQuery = constrainMaxComments( - parseInt((new URLSearchParams(this.props.location.search)).get('max_comments'))) - this.getComments(Math.max(this.props.global.maxComments, maxCommentsQuery), 0) + // The max_comments query parameter can increase the initial comments-to-download + const maxComments = Math.max(this.props.global.maxComments, constrainMaxComments( + parseInt((new URLSearchParams(this.props.location.search)).get('max_comments')))) + + // Get comments starting from the earliest available (not a permalink) + if (commentID === undefined) { + this.contigs.unshift({firstCreated: EARLIEST_CREATED}) + this.getComments(maxComments) + + // Get comments starting from the permalink if possible, otherwise from the earliest available + } else { + this.commentIdAttempts.add(commentID) + getRedditComments([commentID]) + .then(([comment]) => { + this.contigs.unshift({firstCreated: comment?.created_utc || EARLIEST_CREATED}) + this.getComments(maxComments, false, comment) + }) + .catch(() => { + this.contigs.unshift({firstCreated: EARLIEST_CREATED}) + this.getComments(maxComments) + }) + } + } + + // Updates this.curContigIdx based on URL's commentID if it's already downloaded. + // Returns true on success, or false if not found (and then curContigIdx is not updated). + updateCurContig () { + const { commentID } = this.props.match.params + let curContigIdx = -1 + if (commentID === undefined) + curContigIdx = this.contigs[0].firstCreated == EARLIEST_CREATED ? 0 : -1 + else { + const created_utc = this.state.pushshiftCommentLookup.get(commentID)?.created_utc + if (created_utc > EARLIEST_CREATED) + curContigIdx = this.contigs.findIndex(contig => created_utc >= contig.firstCreated && created_utc <= contig.lastCreated) + } + if (curContigIdx < 0) + return false + this.setCurContig(curContigIdx) + return true + } + setCurContig (idx) { + this.curContigIdx = idx + // When the current contig changes, loadedAllComments might also change + const loadedAllComments = Boolean(this.curContig().loadedAllComments) + if (this.state.loadedAllComments != loadedAllComments) + this.setState({loadedAllComments}) } componentDidUpdate () { + + // If the max-to-download Reload button or 'load more comments' was clicked const { loadingMoreComments } = this.props.global.state if (loadingMoreComments) { this.props.global.state.loadingMoreComments = 0 this.setState({reloadingComments: true}) this.props.global.setLoading('Loading more comments from Pushshift...') - this.getComments(loadingMoreComments, this.lastCreatedUtc - 1) + this.updateCurContig() + this.getComments(loadingMoreComments, true) + + // Otherwise if we're loading a comment tree we haven't downloaded yet + } else if (!this.state.loadingComments && !this.state.reloadingComments && !this.updateCurContig()) { + + // If we haven't downloaded from the earliest available yet (not a permalink) + const { commentID } = this.props.match.params + if (commentID === undefined) { + this.setState({loadingComments: true}) + this.props.global.setLoading('Loading comments from Pushshift...') + this.contigs.unshift({firstCreated: EARLIEST_CREATED}) + this.setCurContig(0) + this.getComments(this.props.global.maxComments) + + // If we haven't downloaded this permalink yet + } else if (!this.commentIdAttempts.has(commentID)) { + this.commentIdAttempts.add(commentID) + this.setState({reloadingComments: true}) + this.props.global.setLoading('Loading comments from Pushshift...') + let createdUtcNotFound // true if Reddit doesn't have the comment's created_utc + getRedditComments([commentID]) + .then(([comment]) => { + const created_utc = comment?.created_utc + if (created_utc > EARLIEST_CREATED) { + let insertBefore = this.contigs.findIndex(contig => created_utc < contig.firstCreated) + if (insertBefore == -1) + insertBefore = this.contigs.length + + // If comment isn't inside an existing contig, create a new one and start downloading + if (insertBefore == 0 || created_utc >= this.contigs[insertBefore - 1].lastCreated) { + this.contigs.splice(insertBefore, 0, {firstCreated: created_utc}) + this.setCurContig(insertBefore) + this.getComments(this.props.global.maxComments, false, comment) + + // Otherwise an earlier attempt to download it from Pushshift turned up nothing, + } else { + const { pushshiftCommentLookup } = this.state + this.redditIdsToPushshift(comment) + pushshiftCommentLookup.set(comment.id, comment) // so use the Reddit comment instead + this.setCurContig(insertBefore - 1) // (this was the failed earlier attempt) + this.props.global.setSuccess() + this.setState({pushshiftCommentLookup, loadingComments: false, reloadingComments: false}) + } + } else + createdUtcNotFound = true + }) + .catch(() => createdUtcNotFound = true) + .finally(() => { + if (createdUtcNotFound) { + // As a last resort, try to download starting from the previous contig; + // this only occurs once per commentID due to the commentIdAttempts Set. + if (this.curContigIdx > 0) + this.setCurContig(this.curContigIdx - 1) + // If there is no previous, create one + else if (this.curContig().firstCreated != EARLIEST_CREATED) + this.contigs.unshift({firstCreated: EARLIEST_CREATED}) + this.getComments(this.props.global.maxComments) + } + }) + } } } - getComments (newCommentCount, after) { - const { threadID } = this.props.match.params + // Before calling, either create (and set to current) a new contig to begin downloading + // after a new time, or set the current contig to begin adding to the end of that contig. + // persistent: if true, will try to continue downloading after the current contig has + // been completed and merged with the next contig. + // commentHint: a Reddit comment for use if Pushshift is missing that same comment. + getComments (newCommentCount, persistent = false, commentHint = undefined) { + const { threadID, commentID } = this.props.match.params const { pushshiftCommentLookup } = this.state const redditIdQueue = new ChunkedQueue(redditChunkSize) const pushshiftPromises = [], redditPromises = [] let doRedditComments - // Process a chunk of comments downloaded from Pushshift (started below) + // Process a chunk of comments downloaded from Pushshift (called by getPushshiftComments() below) const processPushshiftComments = comments => { if (comments.length && !this.stopLoading) { pushshiftPromises.push(sleep(0).then(() => { @@ -192,8 +332,10 @@ class Thread extends React.Component { pushshiftCommentLookup.set(id, comment) redditIdQueue.push(id) count++ - if (parent_id != threadID && !pushshiftCommentLookup.has(parent_id)) { - pushshiftCommentLookup.set(parent_id, undefined) + // When viewing the full thread (to prevent false positives), if a parent_id is a comment + // (not a post/thread) and it's missing from Pushshift, try to get it from Reddit instead. + if (commentID === undefined && parent_id != threadID && !pushshiftCommentLookup.has(parent_id)) { + pushshiftCommentLookup.set(parent_id, undefined) // prevents adding it to the Queue multiple times redditIdQueue.push(parent_id) } } @@ -213,9 +355,7 @@ class Thread extends React.Component { let pushshiftComment = pushshiftCommentLookup.get(comment.id) if (pushshiftComment === undefined) { // When a parent comment is missing from pushshift, use the reddit comment instead - comment.parent_id = comment.parent_id.substring(3) - comment.link_id = comment.link_id.substring(3) - pushshiftComment = comment + pushshiftComment = this.redditIdsToPushshift(comment) pushshiftCommentLookup.set(comment.id, pushshiftComment) } else { // Replace pushshift score with reddit (it's usually more accurate) @@ -232,8 +372,7 @@ class Thread extends React.Component { } else if (pushshiftComment !== comment) { if (isRemoved(pushshiftComment.body)) { // If it's deleted in pushshift, but later restored by a mod, use the restored - comment.parent_id = comment.parent_id.substring(3) - comment.link_id = comment.link_id.substring(3) + this.redditIdsToPushshift(comment) pushshiftCommentLookup.set(comment.id, comment) } else if (pushshiftComment.body != comment.body) { pushshiftComment.edited_body = comment.body @@ -249,44 +388,84 @@ class Thread extends React.Component { }) ) - // Download comments from Pushshift, and process each chunk (above) as it's retrieved - getPushshiftComments(processPushshiftComments, threadID, newCommentCount, after) - .then(([lastCreatedUtc, loadedAllComments]) => { - this.lastCreatedUtc = lastCreatedUtc + // Download comments from Pushshift into the current contig, and process each chunk (above) as it's retrieved + const after = this.curContig().lastCreated - 1 || this.curContig().firstCreated - 1 + const before = this.nextContig()?.firstCreated + 1 + getPushshiftComments(processPushshiftComments, threadID, newCommentCount, after, before) + .then(([lastCreatedUtc, curContigLoadedAll]) => { + + // Update the contigs array + if (curContigLoadedAll) { + if (before) { + this.curContig().lastCreated = before - 1 + this.mergeContigs() + } else { + this.curContig().lastCreated = lastCreatedUtc + this.curContig().loadedAllComments = true + } + } else + this.curContig().lastCreated = lastCreatedUtc if (this.stopLoading) return - this.props.global.setLoading('Comparing comments to Reddit API...') - // All comments have been retrieved from Pushshift; wait for processing to finish + // Finished retrieving comments from Pushshift; wait for processing to finish + this.props.global.setLoading('Comparing comments to Reddit API...') Promise.all(pushshiftPromises).then(lengths => { - console.log('Pushshift:', lengths.reduce((a,b) => a+b, 0), 'comments') + const pushshiftComments = lengths.reduce((a,b) => a+b, 0) + console.log('Pushshift:', pushshiftComments, 'comments') + + // If Pushshift didn't find the Reddit commentHint, but should have, use Reddit's comment + if (commentHint && !pushshiftCommentLookup.has(commentHint.id) && + commentHint.created_utc >= this.curContig().firstCreated && ( + commentHint.created_utc < this.curContig().lastCreated || curContigLoadedAll + )) { + this.redditIdsToPushshift(commentHint) + pushshiftCommentLookup.set(commentHint.id, commentHint) + commentHint = undefined + } // All comments from Pushshift have been processed; wait for Reddit to finish while (!redditIdQueue.isEmpty()) doRedditComments(redditIdQueue.shiftChunk()) Promise.all(redditPromises).then(lengths => { console.log('Reddit:', lengths.reduce((a,b) => a+b, 0), 'comments') + if (!this.stopLoading) { - this.props.global.setSuccess() - this.setState({ - pushshiftCommentLookup, - removed: this.state.removed, - deleted: this.state.deleted, - loadedAllComments, - loadingComments: false, - reloadingComments: false - }) + const loadedAllComments = Boolean(this.curContig().loadedAllComments) + if (persistent && !loadedAllComments && pushshiftComments <= newCommentCount - pushshiftChunkSize) + this.getComments(newCommentCount - pushshiftComments, true, commentHint) + + else { + this.props.global.setSuccess() + this.setState({ + pushshiftCommentLookup, + removed: this.state.removed, + deleted: this.state.deleted, + loadedAllComments, + loadingComments: false, + reloadingComments: false + }) + } } }) }) }) - .catch(e => this.props.global.setError(e, e.helpUrl)) + .catch(e => { + this.props.global.setError(e, e.helpUrl) + if (this.curContig().lastCreated === undefined) { + this.contigs.splice(this.curContigIdx, 1) + if (this.curContigIdx >= this.contigs.length) + this.setCurContig(this.contigs.length - 1) + } + }) } render () { const { subreddit, id, author } = this.state.post const { commentID } = this.props.match.params - const reloadingComments = this.state.reloadingComments || this.props.global.state.loadingMoreComments + const reloadingComments = this.state.loadingComments || + this.state.reloadingComments || + this.props.global.state.loadingMoreComments const linkToRestOfComments = `/r/${subreddit}/comments/${id}/_/` const isSingleComment = commentID !== undefined @@ -294,28 +473,32 @@ class Thread extends React.Component { return ( <> - + + + { (!this.state.loadingComments && root) && <> - - {isSingleComment &&
you are viewing a single comment's thread.
- view the rest of the comments + {this.state.reloadingComments ? +
view the rest of the comments
: + view the rest of the comments + }
}