diff --git a/dist/main.css b/dist/main.css index b2e5500..b875672 100644 --- a/dist/main.css +++ b/dist/main.css @@ -1 +1 @@ -html,body{margin:0;padding:0;background-color:#262626;font-family:verdana,arial,helvetica,sans-serif}.wait{cursor:wait}.main{margin:15px;color:#fff}.removed{background-color:#840c09}.deleted{background-color:#00007d}.removed-text{color:#c70300}.deleted-text{color:blue}a{color:#8cb3d9}a:link,a:visited,a:active{text-decoration:none}a:hover{text-decoration:underline}.author{color:#6a98af}.author:not([href]){color:inherit;text-decoration:none}.nowrap{display:inline-block}.space{margin-left:5px}select{background:transparent;border:0;color:#828282}header{background-color:#c70300;display:flex;justify-content:space-between;align-items:center;padding:10px}header #header a{color:#fff}header #header h1{margin:7px 0;text-align:center}header #header h1 a{transition:color .3s}header #header h1 a:hover{text-decoration:none;color:#ff8b88}header #header nav a{margin:0 7px;font-size:20px}header #status{display:flex;justify-content:space-between;align-items:center}header #status #status-text,header #status #status-helpurl{color:#fff;margin:0 20px 0 0}header #status #status-helpurl{font-size:20px}header #status #status-image{height:64px;width:64px;border-radius:32px}#main-box{margin:0 auto;padding:20px;border-radius:5px;color:#ddd;word-wrap:break-word;max-width:800px;background-color:#161616}#main-box h2{margin:20px 0px 12px}#main-box h2.about{margin:0;color:#c70300}#main-box h2.todo{color:#239f2b}#main-box h2.contact{color:#1b767a}#main-box .highlighted{animation:highlighted-anim 3s ease-out}@keyframes highlighted-anim{0%{background-color:#add8e6}}#main-box .question{font-size:18px;color:#add8e6}#main-box ul{padding-left:30px;margin:0}#main-box .bookmarklet{background:#c70300;color:#fff;padding:9px;font-size:large;font-weight:bold;display:inline-block;border-radius:5px;margin:0 7px}.comment{min-width:240px;margin:0 0 8px 0;padding:5px 8px 5px 20px;border:1px solid #333;border-radius:3px;color:#ddd;font-size:14px}.comment .comment-head{font-size:10px}.comment .comment-collapse{cursor:pointer}.comment .comment-collapsed span,.comment .comment-collapsed a:not(.comment-collapse){color:#828282;font-style:oblique}.comment .comment-author{font-weight:bold}.comment .comment-author:not([href]){color:#828282}.comment .comment-poster{color:#ddd;background-color:#0055df;padding:0 2px;border-radius:3px}.comment .comment-score{color:#b4b4b4;font-weight:bold}.comment .comment-time{color:#828282}.comment .comment-body{max-width:840px;line-height:20px}.comment .comment-body p{margin:5px 0;line-height:20px}.comment .comment-body a:hover{text-decoration:none}.comment .comment-links{margin-bottom:6px}.comment .comment-links a{color:#828282;font-weight:bold;font-size:10px;margin-right:4px;cursor:pointer}.comment-odd{background-color:#121212}.comment-even{background-color:#161616}#comment-info{margin-top:10px;font-weight:bold;padding-bottom:5px;border-bottom:1px dotted gray;font-size:16px}#comment-info span{margin-right:8px}#comment-sort,#comment-sort input{color:gray;font-size:12px;margin:5px 0;background-color:#262626}#comment-sort input{border-color:#767676}#comment-sort input[type=number]{width:54px;appearance:textfield}#comment-sort input[type=number]:hover,#comment-sort input[type=number]:focus{appearance:none}#comment-sort input:hover,#comment-sort input:focus,#comment-sort select:hover,#comment-sort select:focus{color:#ddd}#comment-sort option{color:#ddd;background-color:#262626}.thread{display:flex;flex-grow:1;margin-bottom:7px;border-radius:1px;padding-top:6px}.thread .thumbnail{width:70px;margin-right:3px;margin-top:-2px}.thread .thumbnail img{border-radius:1px}.thread .thumbnail-default{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1099px;background-repeat:no-repeat;height:50px}.thread .thumbnail-self{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1267px;background-repeat:no-repeat;height:50px}.thread .thumbnail-image{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1043px;background-repeat:no-repeat;height:50px}.thread .thumbnail-nsfw{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1155px;background-repeat:no-repeat;height:50px}.thread .thread-score-box{margin:0 7px;width:43px}.thread .vote{background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAcCAMAAADC4sagAAABFFBMVEUjJCQiIiIjIyMkIiIkJCQlJSUlJicmJiYnIyEnJycnKCsoKCgpLC8rIyErKyssMDUvLy8wJCAwMDAxMTEyMjI0NDQ2NjY3JR83Nzc5OTk7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NGRkZISEhKSkpKV2tLS0tMTExOTk5PT09QYHdSUlJTU1NVVVVWVlZXV1dXaINYWFhaWlpabYlbW1tecpBgYGBhYWFid5dkZGRle5xlfJ1pgKRrhKhshKlshapuLhZwirFyjbV0j7h4lL97mcZ9MBR/nsyCo9OOMxGcNQ+eNg+kNw6sOA2vOAy0OQy7Ogq/OwrBOwrDPAnLPQjPPQfTPgfbPwXjQQTrQgPzQwLKxGgxAAAAAXRSTlMAQObYZgAAAAlwSFlzAAALEwAACxMBAJqcGAAAAVlJREFUOMu9lF1TwjAQRTcMvmeQOi1QPmxFBQTxW1ERUUFEUVRU/P//w9yGmBTFyZPnpXNmbju76W6IJM8Uw49rLa6Tz4mpq/mCqY3Gvqnj6XQ61lp0mGd8favCwx2tow8wUlpwGWOZrNJKjXO+UVc6fJcMZzV7IswSOU/qej0l0rwyq33wphhAM1kWkcy70GB7mUfUNqG9V02PyPMTMs2Wig5RaTctwzxVXyPqvJh0VkJBIKJ4huncCRAvHOEZyGKegO4a6UDrIdLGCT4C6/QDsE7fA+v0HdAadWqk0SH9D3sgXkm4uJIDYKTFnwwXd3kMrNOnwDp9BmzS5QuTshN1iDRwfkxV9VJTJXLzSWYM+NzEEjWvFE2oGvCE7/2yDeL8riWzM8xmorRazLlNI2rdgNb3ZYLF1Es/t8VE7X6/39Yqlt4tLLwhiLq3XVNLfumP24fo3Opm+wLGCVa252Y8tQAAAABJRU5ErkJggg==);background-repeat:no-repeat;margin-left:auto;margin-right:auto;width:15px;height:14px;cursor:pointer}.thread .upvote{background-position:-15px 0px;margin-top:2px}.thread .downvote{background-position:-15px -14px;margin-top:2px}.thread .thread-score{color:#646464;font-size:13px;font-weight:bold;text-align:center}.thread .thread-content{flex:1;float:left;margin-left:3px}.thread .link-flair{display:inline-block;color:#c8c8c8;background-color:#404040;margin-right:5px;border:1px solid #4d4d4d;border-radius:2px;padding:0 2px;font-size:10px}.thread .thread-title{color:#a6a6a6;margin-right:5px}.thread .thread-title:hover{text-decoration:none}.thread .domain{color:#888;font-size:10px}.thread .thread-selftext{border:1px solid #666;border-radius:7px;margin:5px 0 7px;padding:5px 10px;max-width:840px;color:#ddd;font-size:14px}.thread .thread-selftext p{margin:5px 0;line-height:20px}.thread .thread-selftext a:hover{text-decoration:none}.thread .thread-selftext p:first-child{margin-top:0px}.thread .thread-selftext p:last-child{margin-bottom:0px}.thread .thread-image{max-width:768px;max-height:768px}.thread .thread-info{color:#828282;font-size:10px;margin-top:2px}.thread .total-comments{font-size:10px;font-weight:bold;margin:4px 0}.thread .total-comments a,.thread .total-comments span{color:#828282;margin-right:4px}.thread .total-comments a{cursor:pointer}.view-rest-of-comment{background-color:#264d73;border-color:#2966a3;font-size:13px;padding:10px;color:#ccc;margin-bottom:10px}.load-more{font-weight:bold;font-size:10px}.load-more a,.load-more span{margin-left:16px}.load-more a{cursor:pointer}.load-more .fade{animation:fade-anim 3s 2s ease-in-out forwards}@keyframes fade-anim{100%{color:#262626}}.post-rank{min-width:20px;color:#505050;font-size:16px;text-align:right;margin:14px 4px 0 0} +html,body{margin:0;padding:0;background-color:#262626;font-family:verdana,arial,helvetica,sans-serif}.wait{cursor:wait}.main{margin:15px;color:#fff}.removed{background-color:#840c09}.deleted{background-color:#00007d}.removed-text{color:#c70300}.deleted-text{color:blue}a{color:#8cb3d9}a:link,a:visited,a:active{text-decoration:none}a:hover{text-decoration:underline}.author{color:#6a98af}.author:not([href]){color:inherit;text-decoration:none}.nowrap{display:inline-block}.space{margin-left:5px}select{background:transparent;border:0;color:#828282}header{background-color:#c70300;display:flex;justify-content:space-between;align-items:center;padding:10px}header #header a{color:#fff}header #header h1{margin:7px 0;text-align:center}header #header h1 a{transition:color .3s}header #header h1 a:hover{text-decoration:none;color:#ff8b88}header #header nav a{margin:0 7px;font-size:20px}header #status{display:flex;justify-content:space-between;align-items:center}header #status #status-text,header #status #status-helpurl{color:#fff;margin:0 20px 0 0}header #status #status-helpurl{font-size:20px}header #status #status-image{height:64px;width:64px;border-radius:32px}#main-box{margin:0 auto;padding:20px;border-radius:5px;color:#ddd;word-wrap:break-word;max-width:800px;background-color:#161616}#main-box h2{margin:20px 0px 12px}#main-box h2.about{margin:0;color:#c70300}#main-box h2.todo{color:#239f2b}#main-box h2.contact{color:#1b767a}#main-box .highlighted{animation:highlighted-anim 3s ease-out}@keyframes highlighted-anim{0%{background-color:#add8e6}}#main-box .question{font-size:18px;color:#add8e6}#main-box ul{padding-left:30px;margin:0}#main-box .bookmarklet{background:#c70300;color:#fff;padding:9px;font-size:large;font-weight:bold;display:inline-block;border-radius:5px;margin:0 7px}.comment{min-width:240px;margin:0 0 8px 0;padding:5px 8px 5px 20px;border:1px solid #333;border-radius:3px;color:#ddd;font-size:14px}.comment .comment-head{font-size:10px}.comment .comment-collapse{cursor:pointer}.comment .comment-collapsed span,.comment .comment-collapsed a:not(.comment-collapse){color:#828282;font-style:oblique}.comment .comment-author{font-weight:bold}.comment .comment-author:not([href]){color:#828282}.comment .comment-poster{color:#ddd;background-color:#0055df;padding:0 2px;border-radius:3px}.comment .comment-score{color:#b4b4b4;font-weight:bold}.comment .comment-time{color:#828282}.comment .comment-body{max-width:840px;line-height:20px}.comment .comment-body p{margin:5px 0;line-height:20px}.comment .comment-body a:hover{text-decoration:none}.comment .comment-links{margin-bottom:6px}.comment .comment-links a{color:#828282;font-weight:bold;font-size:10px;margin-right:4px;cursor:pointer}.comment .comment-links a.wait{cursor:wait}.comment .comment-links a.wait:hover{text-decoration:none}.comment-odd{background-color:#121212}.comment-even{background-color:#161616}#comment-info{margin-top:10px;font-weight:bold;padding-bottom:5px;border-bottom:1px dotted gray;font-size:16px}#comment-info span{margin-right:8px}#comment-sort,#comment-sort input{color:gray;font-size:12px;margin:5px 0;background-color:#262626}#comment-sort input{border-color:#767676}#comment-sort input[type=number]{width:54px;appearance:textfield}#comment-sort input[type=number]:hover,#comment-sort input[type=number]:focus{appearance:none}#comment-sort input:hover,#comment-sort input:focus,#comment-sort select:hover,#comment-sort select:focus{color:#ddd}#comment-sort option{color:#ddd;background-color:#262626}.thread{display:flex;flex-grow:1;margin-bottom:7px;border-radius:1px;padding-top:6px}.thread .thumbnail{width:70px;margin-right:3px;margin-top:-2px}.thread .thumbnail img{border-radius:1px}.thread .thumbnail-default{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1099px;background-repeat:no-repeat;height:50px}.thread .thumbnail-self{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1267px;background-repeat:no-repeat;height:50px}.thread .thumbnail-image{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1043px;background-repeat:no-repeat;height:50px}.thread .thumbnail-nsfw{background-image:url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png");background-position:0px -1155px;background-repeat:no-repeat;height:50px}.thread .thread-score-box{margin:0 7px;width:43px}.thread .vote{background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAcCAMAAADC4sagAAABFFBMVEUjJCQiIiIjIyMkIiIkJCQlJSUlJicmJiYnIyEnJycnKCsoKCgpLC8rIyErKyssMDUvLy8wJCAwMDAxMTEyMjI0NDQ2NjY3JR83Nzc5OTk7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NGRkZISEhKSkpKV2tLS0tMTExOTk5PT09QYHdSUlJTU1NVVVVWVlZXV1dXaINYWFhaWlpabYlbW1tecpBgYGBhYWFid5dkZGRle5xlfJ1pgKRrhKhshKlshapuLhZwirFyjbV0j7h4lL97mcZ9MBR/nsyCo9OOMxGcNQ+eNg+kNw6sOA2vOAy0OQy7Ogq/OwrBOwrDPAnLPQjPPQfTPgfbPwXjQQTrQgPzQwLKxGgxAAAAAXRSTlMAQObYZgAAAAlwSFlzAAALEwAACxMBAJqcGAAAAVlJREFUOMu9lF1TwjAQRTcMvmeQOi1QPmxFBQTxW1ERUUFEUVRU/P//w9yGmBTFyZPnpXNmbju76W6IJM8Uw49rLa6Tz4mpq/mCqY3Gvqnj6XQ61lp0mGd8favCwx2tow8wUlpwGWOZrNJKjXO+UVc6fJcMZzV7IswSOU/qej0l0rwyq33wphhAM1kWkcy70GB7mUfUNqG9V02PyPMTMs2Wig5RaTctwzxVXyPqvJh0VkJBIKJ4huncCRAvHOEZyGKegO4a6UDrIdLGCT4C6/QDsE7fA+v0HdAadWqk0SH9D3sgXkm4uJIDYKTFnwwXd3kMrNOnwDp9BmzS5QuTshN1iDRwfkxV9VJTJXLzSWYM+NzEEjWvFE2oGvCE7/2yDeL8riWzM8xmorRazLlNI2rdgNb3ZYLF1Es/t8VE7X6/39Yqlt4tLLwhiLq3XVNLfumP24fo3Opm+wLGCVa252Y8tQAAAABJRU5ErkJggg==);background-repeat:no-repeat;margin-left:auto;margin-right:auto;width:15px;height:14px;cursor:pointer}.thread .upvote{background-position:-15px 0px;margin-top:2px}.thread .downvote{background-position:-15px -14px;margin-top:2px}.thread .thread-score{color:#646464;font-size:13px;font-weight:bold;text-align:center}.thread .thread-content{flex:1;float:left;margin-left:3px}.thread .link-flair{display:inline-block;color:#c8c8c8;background-color:#404040;margin-right:5px;border:1px solid #4d4d4d;border-radius:2px;padding:0 2px;font-size:10px}.thread .thread-title{color:#a6a6a6;margin-right:5px}.thread .thread-title:hover{text-decoration:none}.thread .domain{color:#888;font-size:10px}.thread .thread-selftext{border:1px solid #666;border-radius:7px;margin:5px 0 7px;padding:5px 10px;max-width:840px;color:#ddd;font-size:14px}.thread .thread-selftext p{margin:5px 0;line-height:20px}.thread .thread-selftext a:hover{text-decoration:none}.thread .thread-selftext p:first-child{margin-top:0px}.thread .thread-selftext p:last-child{margin-bottom:0px}.thread .thread-image{max-width:768px;max-height:768px}.thread .thread-info{color:#828282;font-size:10px;margin-top:2px}.thread .total-comments{font-size:10px;font-weight:bold;margin:4px 0}.thread .total-comments a,.thread .total-comments span{color:#828282;margin-right:4px}.thread .total-comments a{cursor:pointer}.view-rest-of-comment{background-color:#264d73;border-color:#2966a3;font-size:13px;padding:10px;color:#ccc;margin-bottom:10px}.view-rest-of-comment .faux-link{color:#8cb3d9}.load-more{font-weight:bold;font-size:10px}.load-more a,.load-more span{margin-left:16px}.load-more a{cursor:pointer}.load-more .fade{animation:fade-anim 3s 5s ease-in-out forwards}@keyframes fade-anim{100%{color:#262626}}.post-rank{min-width:20px;color:#505050;font-size:16px;text-align:right;margin:14px 4px 0 0} diff --git a/src/api/pushshift/index.js b/src/api/pushshift/index.js index 66d55e3..4585a32 100644 --- a/src/api/pushshift/index.js +++ b/src/api/pushshift/index.js @@ -74,7 +74,7 @@ export const getPost = async threadID => { // The callback() function is called with an Array of comments after each chunk is // retrieved. It should return as quickly as possible (scheduling time-taking work // later), and may return false to cause getComments to exit early, or true otherwise. -export const getComments = async (callback, threadID, maxComments, after) => { +export const getComments = async (callback, threadID, maxComments, after = 0, before = undefined) => { let chunks = Math.floor(maxComments / chunkSize), response, lastCreatedUtc = 1 while (true) { @@ -82,7 +82,7 @@ export const getComments = async (callback, threadID, maxComments, after) => { while (true) { await pushshiftTokenBucket.waitForToken() try { - response = await fetchJson(`${commentURL}${threadID}${after ? `&after=${after}` : ''}`) + response = await fetchJson(`${commentURL}${threadID}${after ? `&after=${after}` : ''}${before ? `&before=${before}` : ``}`) break } catch (error) { if (delay >= 8000) // after ~16s of consecutive failures diff --git a/src/pages/common/Post.js b/src/pages/common/Post.js index 3d98971..3a2cd0c 100644 --- a/src/pages/common/Post.js +++ b/src/pages/common/Post.js @@ -86,7 +86,9 @@ export default (props) => { {innerHTML !== undefined &&
}
- {props.num_comments} comments  + {props.reloadingComments ? + {props.num_comments} comments : + {props.num_comments} comments}  reddit  reveddit {props.hasOwnProperty('edited_selftext') && diff --git a/src/pages/thread/Comment.js b/src/pages/thread/Comment.js index 351b89c..ada629f 100644 --- a/src/pages/thread/Comment.js +++ b/src/pages/thread/Comment.js @@ -1,5 +1,5 @@ import React, { useState } from 'react' -import { Link } from 'react-router-dom' +import { Link, NavLink } from 'react-router-dom' import { prettyScore, prettyDate, prettyTimeDiff, exactDateTime, parse, isRemoved } from '../../utils' const Comment = (props) => { @@ -38,6 +38,15 @@ const Comment = (props) => { const [collapsed, setCollapsed] = useState(false) const [showEdited, setShowEdited] = useState(false) const permalink = `/r/${props.subreddit}/comments/${props.link_id}/_/${props.id}/` + const parentlink = props.parent_id == props.link_id ? undefined : ( + props.depth == 0 ? + parent + : + parent + ) return (
@@ -57,7 +66,8 @@ const Comment = (props) => { {prettyScore(props.score)} point{(props.score !== 1) && 's'} - {prettyDate(props.created_utc)} + {props.created_utc && + {prettyDate(props.created_utc)}} {(props.hasOwnProperty('edited_body') || props.edited) && * (last edited {prettyDate(props.edited ? props.edited : props.created_utc)})} @@ -68,6 +78,7 @@ const Comment = (props) => { permalink reddit reveddit + {parentlink} {props.hasOwnProperty('edited_body') && setShowEdited(!showEdited)} onKeyDown={e => e.key == "Enter" && setShowEdited(!showEdited)} diff --git a/src/pages/thread/CommentSection.js b/src/pages/thread/CommentSection.js index c91c201..8f49db7 100644 --- a/src/pages/thread/CommentSection.js +++ b/src/pages/thread/CommentSection.js @@ -6,7 +6,7 @@ import { showRemovedAndDeleted, showRemoved, showDeleted } from '../../utils' -const unflatten = (commentMap, root) => { +const unflatten = (commentMap, rootID, postID) => { const commentTree = [] commentMap.forEach(comment => { @@ -14,28 +14,52 @@ const unflatten = (commentMap, root) => { comment.replies = [] }) - commentMap.forEach(comment => { - if (!comment) - return - const parentID = comment.parent_id - let parentComment + if (rootID == postID) { + commentMap.forEach(comment => { + if (!comment) + return + const parentID = comment.parent_id + if (parentID == postID) + commentTree.push(comment) + else { + const parentComment = commentMap.get(comment.parent_id) + if (parentComment) + parentComment.replies.push(comment) + else + console.warn('Missing parent ID:', parentID, 'for comment', comment) + } + }) + return commentTree - if (parentID === root) { - commentTree.push(comment) - } else if ((parentComment = commentMap.get(parentID)) !== undefined) { - parentComment.replies.push(comment) - } else { - console.warn('Missing parent ID:', parentID, 'for comment', comment) + } else { + const missingRootReplies = [] + commentMap.forEach(comment => { + if (!comment) + return + const parentID = comment.parent_id + const parentComment = commentMap.get(parentID) + if (parentComment) + parentComment.replies.push(comment) + else if (parentID == rootID) + missingRootReplies.push(comment) + }) + let rootComment = commentMap.get(rootID) + if (!rootComment) { + const anyComment = commentMap.values().next().value + if (!anyComment) + return [] + rootComment = { + id: rootID, + link_id: anyComment.link_id, + parent_id: anyComment.link_id, + subreddit: anyComment.subreddit, + score: '?', + body: '...', + replies: missingRootReplies + } } - }) - - let rootComment - if ((rootComment = commentMap.get(root)) !== undefined) { - rootComment.replies = commentTree return [rootComment] } - - return commentTree } const sortCommentTree = (comments, sortFunction) => { @@ -86,7 +110,7 @@ const commentSection = (props) => { ) )) if (needsRebuild) - commentTree = unflatten(props.comments, root) + commentTree = unflatten(props.comments, root, props.postID) if (needsRebuild || commentFilter !== lastFilter) { if (commentFilter === filter.removedDeleted) { diff --git a/src/pages/thread/SortBy.js b/src/pages/thread/SortBy.js index 9c70486..66fe960 100644 --- a/src/pages/thread/SortBy.js +++ b/src/pages/thread/SortBy.js @@ -40,14 +40,14 @@ const sortBy = props => { e.key == "Enter" && e.target.blur()} - onChange= {e => setMaxCommentsField(parseInt(e.target.value))} + onChange= {e => setMaxCommentsField(constrainMaxComments(parseInt(e.target.value)))} onBlur= {e => e.target.value = props.global.setMaxComments(e.target.value)} { ...(isFirefox ? { onClick: e => e.target.focus() } : {}) } defaultValue={props.global.maxComments} type='number' maxLength='5' required min={minCommentsLimit} max={maxCommentsLimit} step={minCommentsLimit} /> - { !props.loadedAllComments && !props.reloadingComments && constrainMaxComments(maxCommentsField) - minCommentsLimit >= props.total && + { !props.reloadingComments && !props.loadedAllComments && maxCommentsField > props.global.maxComments && maxCommentsField - minCommentsLimit >= props.total && props.global.loadMoreComments(props.global.maxComments - props.total)} type='button' value='Reload' /> diff --git a/src/pages/thread/index.js b/src/pages/thread/index.js index 1e48768..02ec736 100644 --- a/src/pages/thread/index.js +++ b/src/pages/thread/index.js @@ -7,7 +7,8 @@ import { } from '../../api/reddit' import { getPost as getPushshiftPost, - getComments as getPushshiftComments + getComments as getPushshiftComments, + chunkSize as pushshiftChunkSize } from '../../api/pushshift' import { isDeleted, isRemoved, sleep } from '../../utils' import { connect, constrainMaxComments } from '../../state' @@ -47,6 +48,9 @@ class ChunkedQueue { } } +// The .firstCreated of the contig containing a post's first comment (see contigs below) +const EARLIEST_CREATED = 1 + class Thread extends React.Component { state = { post: {}, @@ -58,8 +62,33 @@ class Thread extends React.Component { reloadingComments: false } + // A 'contig' is an object representing a contiguous block of comments currently being downloaded or already + // downloaded, e.g. { firstCreated: #, lastCreated: # } (secs past the epoch; min. value of EARLIEST_CREATED) + contigs = [] // sorted non-overlapping array of contig objects + curContigIdx = 0 + curContig () { return this.contigs[this.curContigIdx] } + nextContig () { return this.contigs[this.curContigIdx + 1] } + + // If the current contig and the next probably overlap, merge them + // (should only be called if there's another reason to believe they overlap) + mergeContigs () { + const nextContig = this.nextContig() + if (this.curContig().lastCreated >= nextContig?.firstCreated) // probably; definitely would be '>' + nextContig.firstCreated = this.contigs.splice(this.curContigIdx, 1)[0].firstCreated + else + console.warn("Can't merge contigs", this.curContig(), "and", nextContig) // shouldn't happen + } + + redditIdsToPushshift (comment) { + comment.parent_id = comment.parent_id?.substring(3) || this.props.match.params.threadID + comment.link_id = comment.link_id?.substring(3) || this.props.match.params.threadID + return comment + } + + commentIdAttempts = new Set() // keeps track of attempts to load permalinks to avoid reattempts + componentDidMount () { - const { subreddit, threadID } = this.props.match.params + const { subreddit, threadID, commentID } = this.props.match.params this.props.global.setLoading('Loading post...') // Get post from Reddit. Each code path below should end in either @@ -159,29 +188,140 @@ class Thread extends React.Component { } }) - const maxCommentsQuery = constrainMaxComments( - parseInt((new URLSearchParams(this.props.location.search)).get('max_comments'))) - this.getComments(Math.max(this.props.global.maxComments, maxCommentsQuery), 0) + // The max_comments query parameter can increase the initial comments-to-download + const maxComments = Math.max(this.props.global.maxComments, constrainMaxComments( + parseInt((new URLSearchParams(this.props.location.search)).get('max_comments')))) + + // Get comments starting from the earliest available (not a permalink) + if (commentID === undefined) { + this.contigs.unshift({firstCreated: EARLIEST_CREATED}) + this.getComments(maxComments) + + // Get comments starting from the permalink if possible, otherwise from the earliest available + } else { + this.commentIdAttempts.add(commentID) + getRedditComments([commentID]) + .then(([comment]) => { + this.contigs.unshift({firstCreated: comment?.created_utc || EARLIEST_CREATED}) + this.getComments(maxComments, false, comment) + }) + .catch(() => { + this.contigs.unshift({firstCreated: EARLIEST_CREATED}) + this.getComments(maxComments) + }) + } + } + + // Updates this.curContigIdx based on URL's commentID if it's already downloaded. + // Returns true on success, or false if not found (and then curContigIdx is not updated). + updateCurContig () { + const { commentID } = this.props.match.params + let curContigIdx = -1 + if (commentID === undefined) + curContigIdx = this.contigs[0].firstCreated == EARLIEST_CREATED ? 0 : -1 + else { + const created_utc = this.state.pushshiftCommentLookup.get(commentID)?.created_utc + if (created_utc > EARLIEST_CREATED) + curContigIdx = this.contigs.findIndex(contig => created_utc >= contig.firstCreated && created_utc <= contig.lastCreated) + } + if (curContigIdx < 0) + return false + this.setCurContig(curContigIdx) + return true + } + setCurContig (idx) { + this.curContigIdx = idx + // When the current contig changes, loadedAllComments might also change + const loadedAllComments = Boolean(this.curContig().loadedAllComments) + if (this.state.loadedAllComments != loadedAllComments) + this.setState({loadedAllComments}) } componentDidUpdate () { + + // If the max-to-download Reload button or 'load more comments' was clicked const { loadingMoreComments } = this.props.global.state if (loadingMoreComments) { this.props.global.state.loadingMoreComments = 0 this.setState({reloadingComments: true}) this.props.global.setLoading('Loading more comments from Pushshift...') - this.getComments(loadingMoreComments, this.lastCreatedUtc - 1) + this.updateCurContig() + this.getComments(loadingMoreComments, true) + + // Otherwise if we're loading a comment tree we haven't downloaded yet + } else if (!this.state.loadingComments && !this.state.reloadingComments && !this.updateCurContig()) { + + // If we haven't downloaded from the earliest available yet (not a permalink) + const { commentID } = this.props.match.params + if (commentID === undefined) { + this.setState({loadingComments: true}) + this.props.global.setLoading('Loading comments from Pushshift...') + this.contigs.unshift({firstCreated: EARLIEST_CREATED}) + this.setCurContig(0) + this.getComments(this.props.global.maxComments) + + // If we haven't downloaded this permalink yet + } else if (!this.commentIdAttempts.has(commentID)) { + this.commentIdAttempts.add(commentID) + this.setState({reloadingComments: true}) + this.props.global.setLoading('Loading comments from Pushshift...') + let createdUtcNotFound // true if Reddit doesn't have the comment's created_utc + getRedditComments([commentID]) + .then(([comment]) => { + const created_utc = comment?.created_utc + if (created_utc > EARLIEST_CREATED) { + let insertBefore = this.contigs.findIndex(contig => created_utc < contig.firstCreated) + if (insertBefore == -1) + insertBefore = this.contigs.length + + // If comment isn't inside an existing contig, create a new one and start downloading + if (insertBefore == 0 || created_utc >= this.contigs[insertBefore - 1].lastCreated) { + this.contigs.splice(insertBefore, 0, {firstCreated: created_utc}) + this.setCurContig(insertBefore) + this.getComments(this.props.global.maxComments, false, comment) + + // Otherwise an earlier attempt to download it from Pushshift turned up nothing, + } else { + const { pushshiftCommentLookup } = this.state + this.redditIdsToPushshift(comment) + pushshiftCommentLookup.set(comment.id, comment) // so use the Reddit comment instead + this.setCurContig(insertBefore - 1) // (this was the failed earlier attempt) + this.props.global.setSuccess() + this.setState({pushshiftCommentLookup, loadingComments: false, reloadingComments: false}) + } + } else + createdUtcNotFound = true + }) + .catch(() => createdUtcNotFound = true) + .finally(() => { + if (createdUtcNotFound) { + // As a last resort, try to download starting from the previous contig; + // this only occurs once per commentID due to the commentIdAttempts Set. + if (this.curContigIdx > 0) + this.setCurContig(this.curContigIdx - 1) + // If there is no previous, create one + else if (this.curContig().firstCreated != EARLIEST_CREATED) + this.contigs.unshift({firstCreated: EARLIEST_CREATED}) + this.getComments(this.props.global.maxComments) + } + }) + } } } - getComments (newCommentCount, after) { - const { threadID } = this.props.match.params + // Before calling, either create (and set to current) a new contig to begin downloading + // after a new time, or set the current contig to begin adding to the end of that contig. + // persistent: if true, will try to continue downloading after the current contig has + // been completed and merged with the next contig. + // commentHint: a Reddit comment for use if Pushshift is missing that same comment. + getComments (newCommentCount, persistent = false, commentHint = undefined) { + const { threadID, commentID } = this.props.match.params const { pushshiftCommentLookup } = this.state const redditIdQueue = new ChunkedQueue(redditChunkSize) const pushshiftPromises = [], redditPromises = [] let doRedditComments - // Process a chunk of comments downloaded from Pushshift (started below) + // Process a chunk of comments downloaded from Pushshift (called by getPushshiftComments() below) const processPushshiftComments = comments => { if (comments.length && !this.stopLoading) { pushshiftPromises.push(sleep(0).then(() => { @@ -192,8 +332,10 @@ class Thread extends React.Component { pushshiftCommentLookup.set(id, comment) redditIdQueue.push(id) count++ - if (parent_id != threadID && !pushshiftCommentLookup.has(parent_id)) { - pushshiftCommentLookup.set(parent_id, undefined) + // When viewing the full thread (to prevent false positives), if a parent_id is a comment + // (not a post/thread) and it's missing from Pushshift, try to get it from Reddit instead. + if (commentID === undefined && parent_id != threadID && !pushshiftCommentLookup.has(parent_id)) { + pushshiftCommentLookup.set(parent_id, undefined) // prevents adding it to the Queue multiple times redditIdQueue.push(parent_id) } } @@ -213,9 +355,7 @@ class Thread extends React.Component { let pushshiftComment = pushshiftCommentLookup.get(comment.id) if (pushshiftComment === undefined) { // When a parent comment is missing from pushshift, use the reddit comment instead - comment.parent_id = comment.parent_id.substring(3) - comment.link_id = comment.link_id.substring(3) - pushshiftComment = comment + pushshiftComment = this.redditIdsToPushshift(comment) pushshiftCommentLookup.set(comment.id, pushshiftComment) } else { // Replace pushshift score with reddit (it's usually more accurate) @@ -232,8 +372,7 @@ class Thread extends React.Component { } else if (pushshiftComment !== comment) { if (isRemoved(pushshiftComment.body)) { // If it's deleted in pushshift, but later restored by a mod, use the restored - comment.parent_id = comment.parent_id.substring(3) - comment.link_id = comment.link_id.substring(3) + this.redditIdsToPushshift(comment) pushshiftCommentLookup.set(comment.id, comment) } else if (pushshiftComment.body != comment.body) { pushshiftComment.edited_body = comment.body @@ -249,44 +388,84 @@ class Thread extends React.Component { }) ) - // Download comments from Pushshift, and process each chunk (above) as it's retrieved - getPushshiftComments(processPushshiftComments, threadID, newCommentCount, after) - .then(([lastCreatedUtc, loadedAllComments]) => { - this.lastCreatedUtc = lastCreatedUtc + // Download comments from Pushshift into the current contig, and process each chunk (above) as it's retrieved + const after = this.curContig().lastCreated - 1 || this.curContig().firstCreated - 1 + const before = this.nextContig()?.firstCreated + 1 + getPushshiftComments(processPushshiftComments, threadID, newCommentCount, after, before) + .then(([lastCreatedUtc, curContigLoadedAll]) => { + + // Update the contigs array + if (curContigLoadedAll) { + if (before) { + this.curContig().lastCreated = before - 1 + this.mergeContigs() + } else { + this.curContig().lastCreated = lastCreatedUtc + this.curContig().loadedAllComments = true + } + } else + this.curContig().lastCreated = lastCreatedUtc if (this.stopLoading) return - this.props.global.setLoading('Comparing comments to Reddit API...') - // All comments have been retrieved from Pushshift; wait for processing to finish + // Finished retrieving comments from Pushshift; wait for processing to finish + this.props.global.setLoading('Comparing comments to Reddit API...') Promise.all(pushshiftPromises).then(lengths => { - console.log('Pushshift:', lengths.reduce((a,b) => a+b, 0), 'comments') + const pushshiftComments = lengths.reduce((a,b) => a+b, 0) + console.log('Pushshift:', pushshiftComments, 'comments') + + // If Pushshift didn't find the Reddit commentHint, but should have, use Reddit's comment + if (commentHint && !pushshiftCommentLookup.has(commentHint.id) && + commentHint.created_utc >= this.curContig().firstCreated && ( + commentHint.created_utc < this.curContig().lastCreated || curContigLoadedAll + )) { + this.redditIdsToPushshift(commentHint) + pushshiftCommentLookup.set(commentHint.id, commentHint) + commentHint = undefined + } // All comments from Pushshift have been processed; wait for Reddit to finish while (!redditIdQueue.isEmpty()) doRedditComments(redditIdQueue.shiftChunk()) Promise.all(redditPromises).then(lengths => { console.log('Reddit:', lengths.reduce((a,b) => a+b, 0), 'comments') + if (!this.stopLoading) { - this.props.global.setSuccess() - this.setState({ - pushshiftCommentLookup, - removed: this.state.removed, - deleted: this.state.deleted, - loadedAllComments, - loadingComments: false, - reloadingComments: false - }) + const loadedAllComments = Boolean(this.curContig().loadedAllComments) + if (persistent && !loadedAllComments && pushshiftComments <= newCommentCount - pushshiftChunkSize) + this.getComments(newCommentCount - pushshiftComments, true, commentHint) + + else { + this.props.global.setSuccess() + this.setState({ + pushshiftCommentLookup, + removed: this.state.removed, + deleted: this.state.deleted, + loadedAllComments, + loadingComments: false, + reloadingComments: false + }) + } } }) }) }) - .catch(e => this.props.global.setError(e, e.helpUrl)) + .catch(e => { + this.props.global.setError(e, e.helpUrl) + if (this.curContig().lastCreated === undefined) { + this.contigs.splice(this.curContigIdx, 1) + if (this.curContigIdx >= this.contigs.length) + this.setCurContig(this.contigs.length - 1) + } + }) } render () { const { subreddit, id, author } = this.state.post const { commentID } = this.props.match.params - const reloadingComments = this.state.reloadingComments || this.props.global.state.loadingMoreComments + const reloadingComments = this.state.loadingComments || + this.state.reloadingComments || + this.props.global.state.loadingMoreComments const linkToRestOfComments = `/r/${subreddit}/comments/${id}/_/` const isSingleComment = commentID !== undefined @@ -294,28 +473,32 @@ class Thread extends React.Component { return ( <> - + + + { (!this.state.loadingComments && root) && <> - - {isSingleComment &&
you are viewing a single comment's thread.
- view the rest of the comments + {this.state.reloadingComments ? +
view the rest of the comments
: + view the rest of the comments + }
}