prehistoric-systems/removeddit
1
0
Fork 0
mirror of https://github.com/gurnec/removeddit.git synced 2026-03-11 08:54:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve permalinks and add parent link to comments

Browse source 
Beforehand, visiting a permalink would cause Unddit to begin loading
from the Post's first comment. If the max-to-download setting was too
small, and the permalinked comment wasn't yet encountered, no comment
tree would be displayed.

Now, only comments that occur at the same time as the permalinked
comment or later (chronologically) are downloaded.

This also adds a Parent link to each comment. When viewing a permalinked
comment, clicking the top-level Parent link will download only new
comments between the parent and child comments. Likewise, clicking the
'view rest of comments' link will only download new comments from the
first comment up until the previously viewed comment.

In general, Unddit will not re-download comments that it knows it has
previously downloaded, nor download more than max-to-download comments
per link clicked. This can leave 'gaps' in the list of downloaded
comments. If the 'load more comments' link is clicked, Unddit will only
download comments that can fill such gaps between the currently-linked
permalink (or if not viewing one, the first comment) and the last, up to
the max-to-download setting.
This commit is contained in:
Christopher Gurnee 2022-03-18 18:10:20 +00:00
parent 2d436062ce
commit 0abf9222fb
9 changed files with 304 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
dist/main.css vendored
View file

File diff suppressed because one or more lines are too long

4
src/api/pushshift/index.js
View file

@ -74,7 +74,7 @@ export const getPost = async threadID => {
// The callback() function is called with an Array of comments after each chunk is
// retrieved. It should return as quickly as possible (scheduling time-taking work
// later), and may return false to cause getComments to exit early, or true otherwise.
export const getComments = async (callback, threadID, maxComments, after) => {
export const getComments = async (callback, threadID, maxComments, after = 0, before = undefined) => {
let chunks = Math.floor(maxComments / chunkSize), response, lastCreatedUtc = 1
while (true) {
@ -82,7 +82,7 @@ export const getComments = async (callback, threadID, maxComments, after) => {
while (true) {
await pushshiftTokenBucket.waitForToken()
try {
response = await fetchJson(`${commentURL}${threadID}${after ? `&after=${after}` : ''}`)
response = await fetchJson(`${commentURL}${threadID}${after ? `&after=${after}` : ''}${before ? `&before=${before}` : ``}`)
break
} catch (error) {
if (delay >= 8000) // after ~16s of consecutive failures

4
src/pages/common/Post.js
View file

@ -86,7 +86,9 @@ export default (props) => {
{innerHTML !== undefined &&
<div className='thread-selftext user-text' dangerouslySetInnerHTML={{ __html: showEdited ? editedInnerHTML : innerHTML }} />}
<div className='total-comments'>
<Link to={props.permalink}>{props.num_comments} comments</Link>&nbsp;
{props.reloadingComments ?
<span>{props.num_comments} comments</span> :
<Link to={props.permalink}>{props.num_comments} comments</Link>}&nbsp;
<a href={`https://www.reddit.com${props.permalink}`}>reddit</a>&nbsp;
<a href={`https://reveddit.com${props.permalink}`}>reveddit</a>
{props.hasOwnProperty('edited_selftext') &&

15
src/pages/thread/Comment.js
View file

@ -1,5 +1,5 @@
import React, { useState } from 'react'
import { Link } from 'react-router-dom'
import { Link, NavLink } from 'react-router-dom'
import { prettyScore, prettyDate, prettyTimeDiff, exactDateTime, parse, isRemoved } from '../../utils'
const Comment = (props) => {
@ -38,6 +38,15 @@ const Comment = (props) => {
const [collapsed, setCollapsed] = useState(false)
const [showEdited, setShowEdited] = useState(false)
const permalink = `/r/${props.subreddit}/comments/${props.link_id}/_/${props.id}/`
const parentlink = props.parent_id == props.link_id ? undefined : (
props.depth == 0 ?
<NavLink
to={`/r/${props.subreddit}/comments/${props.link_id}/_/${props.parent_id}/`}
activeClassName='wait'
>parent</NavLink>
:
<a href={`#${props.parent_id}`}>parent</a>
)
return (
<div id={props.id} className={commentStyle}>
@ -57,7 +66,8 @@ const Comment = (props) => {
<span className='space' />
<span className='comment-score'>{prettyScore(props.score)} point{(props.score !== 1) && 's'}</span>
<span className='space' />
<span className='comment-time' title={exactDateTime(props.created_utc)}>{prettyDate(props.created_utc)}</span>
{props.created_utc &&
<span className='comment-time' title={exactDateTime(props.created_utc)}>{prettyDate(props.created_utc)}</span>}
{(props.hasOwnProperty('edited_body') || props.edited) &&
<span className='comment-time' title={props.edited ? exactDateTime(props.edited) : 'within 3 minutes'}
>* (last edited {prettyDate(props.edited ? props.edited : props.created_utc)})</span>}
@ -68,6 +78,7 @@ const Comment = (props) => {
<Link to={permalink}>permalink</Link>
<a href={`https://www.reddit.com${permalink}`}>reddit</a>
<a href={`https://reveddit.com${permalink}`}>reveddit</a>
{parentlink}
{props.hasOwnProperty('edited_body') &&
<a onClick= {() => setShowEdited(!showEdited)}
onKeyDown={e => e.key == "Enter" && setShowEdited(!showEdited)}

64
src/pages/thread/CommentSection.js
View file

@ -6,7 +6,7 @@ import {
showRemovedAndDeleted, showRemoved, showDeleted
} from '../../utils'
const unflatten = (commentMap, root) => {
const unflatten = (commentMap, rootID, postID) => {
const commentTree = []
commentMap.forEach(comment => {
@ -14,28 +14,52 @@ const unflatten = (commentMap, root) => {
comment.replies = []
})
commentMap.forEach(comment => {
if (!comment)
return
const parentID = comment.parent_id
let parentComment
if (rootID == postID) {
commentMap.forEach(comment => {
if (!comment)
return
const parentID = comment.parent_id
if (parentID == postID)
commentTree.push(comment)
else {
const parentComment = commentMap.get(comment.parent_id)
if (parentComment)
parentComment.replies.push(comment)
else
console.warn('Missing parent ID:', parentID, 'for comment', comment)
}
})
return commentTree
if (parentID === root) {
commentTree.push(comment)
} else if ((parentComment = commentMap.get(parentID)) !== undefined) {
parentComment.replies.push(comment)
} else {
console.warn('Missing parent ID:', parentID, 'for comment', comment)
} else {
const missingRootReplies = []
commentMap.forEach(comment => {
if (!comment)
return
const parentID = comment.parent_id
const parentComment = commentMap.get(parentID)
if (parentComment)
parentComment.replies.push(comment)
else if (parentID == rootID)
missingRootReplies.push(comment)
})
let rootComment = commentMap.get(rootID)
if (!rootComment) {
const anyComment = commentMap.values().next().value
if (!anyComment)
return []
rootComment = {
id: rootID,
link_id: anyComment.link_id,
parent_id: anyComment.link_id,
subreddit: anyComment.subreddit,
score: '?',
body: '...',
replies: missingRootReplies
}
}
})
let rootComment
if ((rootComment = commentMap.get(root)) !== undefined) {
rootComment.replies = commentTree
return [rootComment]
}
return commentTree
}
const sortCommentTree = (comments, sortFunction) => {
@ -86,7 +110,7 @@ const commentSection = (props) => {
)
))
if (needsRebuild)
commentTree = unflatten(props.comments, root)
commentTree = unflatten(props.comments, root, props.postID)
if (needsRebuild || commentFilter !== lastFilter) {
if (commentFilter === filter.removedDeleted) {

4
src/pages/thread/SortBy.js
View file

@ -40,14 +40,14 @@ const sortBy = props => {
<span className='space' />
<input id='maxComments'
onKeyDown={e => e.key == "Enter" && e.target.blur()}
onChange= {e => setMaxCommentsField(parseInt(e.target.value))}
onChange= {e => setMaxCommentsField(constrainMaxComments(parseInt(e.target.value)))}
onBlur= {e => e.target.value = props.global.setMaxComments(e.target.value)}
{ ...(isFirefox ? {
onClick: e => e.target.focus() } : {}) }
defaultValue={props.global.maxComments} type='number' maxLength='5' required
min={minCommentsLimit} max={maxCommentsLimit} step={minCommentsLimit} />
</span>
{ !props.loadedAllComments && !props.reloadingComments && constrainMaxComments(maxCommentsField) - minCommentsLimit >= props.total &&
{ !props.reloadingComments && !props.loadedAllComments && maxCommentsField > props.global.maxComments && maxCommentsField - minCommentsLimit >= props.total &&
<span className='nowrap'>
<span className='space' />
<input onClick={() => props.global.loadMoreComments(props.global.maxComments - props.total)} type='button' value='Reload' />

275
src/pages/thread/index.js
View file

@ -7,7 +7,8 @@ import {
} from '../../api/reddit'
import {
getPost as getPushshiftPost,
getComments as getPushshiftComments
getComments as getPushshiftComments,
chunkSize as pushshiftChunkSize
} from '../../api/pushshift'
import { isDeleted, isRemoved, sleep } from '../../utils'
import { connect, constrainMaxComments } from '../../state'
@ -47,6 +48,9 @@ class ChunkedQueue {
}
}
// The .firstCreated of the contig containing a post's first comment (see contigs below)
const EARLIEST_CREATED = 1
class Thread extends React.Component {
state = {
post: {},
@ -58,8 +62,33 @@ class Thread extends React.Component {
reloadingComments: false
}
// A 'contig' is an object representing a contiguous block of comments currently being downloaded or already
// downloaded, e.g. { firstCreated: #, lastCreated: # } (secs past the epoch; min. value of EARLIEST_CREATED)
contigs = [] // sorted non-overlapping array of contig objects
curContigIdx = 0
curContig () { return this.contigs[this.curContigIdx] }
nextContig () { return this.contigs[this.curContigIdx + 1] }
// If the current contig and the next probably overlap, merge them
// (should only be called if there's another reason to believe they overlap)
mergeContigs () {
const nextContig = this.nextContig()
if (this.curContig().lastCreated >= nextContig?.firstCreated) // probably; definitely would be '>'
nextContig.firstCreated = this.contigs.splice(this.curContigIdx, 1)[0].firstCreated
else
console.warn("Can't merge contigs", this.curContig(), "and", nextContig) // shouldn't happen
}
redditIdsToPushshift (comment) {
comment.parent_id = comment.parent_id?.substring(3) || this.props.match.params.threadID
comment.link_id = comment.link_id?.substring(3) || this.props.match.params.threadID
return comment
}
commentIdAttempts = new Set() // keeps track of attempts to load permalinks to avoid reattempts
componentDidMount () {
const { subreddit, threadID } = this.props.match.params
const { subreddit, threadID, commentID } = this.props.match.params
this.props.global.setLoading('Loading post...')
// Get post from Reddit. Each code path below should end in either
@ -159,29 +188,140 @@ class Thread extends React.Component {
}
})
const maxCommentsQuery = constrainMaxComments(
parseInt((new URLSearchParams(this.props.location.search)).get('max_comments')))
this.getComments(Math.max(this.props.global.maxComments, maxCommentsQuery), 0)
// The max_comments query parameter can increase the initial comments-to-download
const maxComments = Math.max(this.props.global.maxComments, constrainMaxComments(
parseInt((new URLSearchParams(this.props.location.search)).get('max_comments'))))
// Get comments starting from the earliest available (not a permalink)
if (commentID === undefined) {
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
this.getComments(maxComments)
// Get comments starting from the permalink if possible, otherwise from the earliest available
} else {
this.commentIdAttempts.add(commentID)
getRedditComments([commentID])
.then(([comment]) => {
this.contigs.unshift({firstCreated: comment?.created_utc || EARLIEST_CREATED})
this.getComments(maxComments, false, comment)
})
.catch(() => {
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
this.getComments(maxComments)
})
}
}
// Updates this.curContigIdx based on URL's commentID if it's already downloaded.
// Returns true on success, or false if not found (and then curContigIdx is not updated).
updateCurContig () {
const { commentID } = this.props.match.params
let curContigIdx = -1
if (commentID === undefined)
curContigIdx = this.contigs[0].firstCreated == EARLIEST_CREATED ? 0 : -1
else {
const created_utc = this.state.pushshiftCommentLookup.get(commentID)?.created_utc
if (created_utc > EARLIEST_CREATED)
curContigIdx = this.contigs.findIndex(contig => created_utc >= contig.firstCreated && created_utc <= contig.lastCreated)
}
if (curContigIdx < 0)
return false
this.setCurContig(curContigIdx)
return true
}
setCurContig (idx) {
this.curContigIdx = idx
// When the current contig changes, loadedAllComments might also change
const loadedAllComments = Boolean(this.curContig().loadedAllComments)
if (this.state.loadedAllComments != loadedAllComments)
this.setState({loadedAllComments})
}
componentDidUpdate () {
// If the max-to-download Reload button or 'load more comments' was clicked
const { loadingMoreComments } = this.props.global.state
if (loadingMoreComments) {
this.props.global.state.loadingMoreComments = 0
this.setState({reloadingComments: true})
this.props.global.setLoading('Loading more comments from Pushshift...')
this.getComments(loadingMoreComments, this.lastCreatedUtc - 1)
this.updateCurContig()
this.getComments(loadingMoreComments, true)
// Otherwise if we're loading a comment tree we haven't downloaded yet
} else if (!this.state.loadingComments && !this.state.reloadingComments && !this.updateCurContig()) {
// If we haven't downloaded from the earliest available yet (not a permalink)
const { commentID } = this.props.match.params
if (commentID === undefined) {
this.setState({loadingComments: true})
this.props.global.setLoading('Loading comments from Pushshift...')
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
this.setCurContig(0)
this.getComments(this.props.global.maxComments)
// If we haven't downloaded this permalink yet
} else if (!this.commentIdAttempts.has(commentID)) {
this.commentIdAttempts.add(commentID)
this.setState({reloadingComments: true})
this.props.global.setLoading('Loading comments from Pushshift...')
let createdUtcNotFound // true if Reddit doesn't have the comment's created_utc
getRedditComments([commentID])
.then(([comment]) => {
const created_utc = comment?.created_utc
if (created_utc > EARLIEST_CREATED) {
let insertBefore = this.contigs.findIndex(contig => created_utc < contig.firstCreated)
if (insertBefore == -1)
insertBefore = this.contigs.length
// If comment isn't inside an existing contig, create a new one and start downloading
if (insertBefore == 0 || created_utc >= this.contigs[insertBefore - 1].lastCreated) {
this.contigs.splice(insertBefore, 0, {firstCreated: created_utc})
this.setCurContig(insertBefore)
this.getComments(this.props.global.maxComments, false, comment)
// Otherwise an earlier attempt to download it from Pushshift turned up nothing,
} else {
const { pushshiftCommentLookup } = this.state
this.redditIdsToPushshift(comment)
pushshiftCommentLookup.set(comment.id, comment) // so use the Reddit comment instead
this.setCurContig(insertBefore - 1) // (this was the failed earlier attempt)
this.props.global.setSuccess()
this.setState({pushshiftCommentLookup, loadingComments: false, reloadingComments: false})
}
} else
createdUtcNotFound = true
})
.catch(() => createdUtcNotFound = true)
.finally(() => {
if (createdUtcNotFound) {
// As a last resort, try to download starting from the previous contig;
// this only occurs once per commentID due to the commentIdAttempts Set.
if (this.curContigIdx > 0)
this.setCurContig(this.curContigIdx - 1)
// If there is no previous, create one
else if (this.curContig().firstCreated != EARLIEST_CREATED)
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
this.getComments(this.props.global.maxComments)
}
})
}
}
}
getComments (newCommentCount, after) {
const { threadID } = this.props.match.params
// Before calling, either create (and set to current) a new contig to begin downloading
// after a new time, or set the current contig to begin adding to the end of that contig.
// persistent: if true, will try to continue downloading after the current contig has
// been completed and merged with the next contig.
// commentHint: a Reddit comment for use if Pushshift is missing that same comment.
getComments (newCommentCount, persistent = false, commentHint = undefined) {
const { threadID, commentID } = this.props.match.params
const { pushshiftCommentLookup } = this.state
const redditIdQueue = new ChunkedQueue(redditChunkSize)
const pushshiftPromises = [], redditPromises = []
let doRedditComments
// Process a chunk of comments downloaded from Pushshift (started below)
// Process a chunk of comments downloaded from Pushshift (called by getPushshiftComments() below)
const processPushshiftComments = comments => {
if (comments.length && !this.stopLoading) {
pushshiftPromises.push(sleep(0).then(() => {
@ -192,8 +332,10 @@ class Thread extends React.Component {
pushshiftCommentLookup.set(id, comment)
redditIdQueue.push(id)
count++
if (parent_id != threadID && !pushshiftCommentLookup.has(parent_id)) {
pushshiftCommentLookup.set(parent_id, undefined)
// When viewing the full thread (to prevent false positives), if a parent_id is a comment
// (not a post/thread) and it's missing from Pushshift, try to get it from Reddit instead.
if (commentID === undefined && parent_id != threadID && !pushshiftCommentLookup.has(parent_id)) {
pushshiftCommentLookup.set(parent_id, undefined) // prevents adding it to the Queue multiple times
redditIdQueue.push(parent_id)
}
}
@ -213,9 +355,7 @@ class Thread extends React.Component {
let pushshiftComment = pushshiftCommentLookup.get(comment.id)
if (pushshiftComment === undefined) {
// When a parent comment is missing from pushshift, use the reddit comment instead
comment.parent_id = comment.parent_id.substring(3)
comment.link_id = comment.link_id.substring(3)
pushshiftComment = comment
pushshiftComment = this.redditIdsToPushshift(comment)
pushshiftCommentLookup.set(comment.id, pushshiftComment)
} else {
// Replace pushshift score with reddit (it's usually more accurate)
@ -232,8 +372,7 @@ class Thread extends React.Component {
} else if (pushshiftComment !== comment) {
if (isRemoved(pushshiftComment.body)) {
// If it's deleted in pushshift, but later restored by a mod, use the restored
comment.parent_id = comment.parent_id.substring(3)
comment.link_id = comment.link_id.substring(3)
this.redditIdsToPushshift(comment)
pushshiftCommentLookup.set(comment.id, comment)
} else if (pushshiftComment.body != comment.body) {
pushshiftComment.edited_body = comment.body
@ -249,44 +388,84 @@ class Thread extends React.Component {
})
)
// Download comments from Pushshift, and process each chunk (above) as it's retrieved
getPushshiftComments(processPushshiftComments, threadID, newCommentCount, after)
.then(([lastCreatedUtc, loadedAllComments]) => {
this.lastCreatedUtc = lastCreatedUtc
// Download comments from Pushshift into the current contig, and process each chunk (above) as it's retrieved
const after = this.curContig().lastCreated - 1 || this.curContig().firstCreated - 1
const before = this.nextContig()?.firstCreated + 1
getPushshiftComments(processPushshiftComments, threadID, newCommentCount, after, before)
.then(([lastCreatedUtc, curContigLoadedAll]) => {
// Update the contigs array
if (curContigLoadedAll) {
if (before) {
this.curContig().lastCreated = before - 1
this.mergeContigs()
} else {
this.curContig().lastCreated = lastCreatedUtc
this.curContig().loadedAllComments = true
}
} else
this.curContig().lastCreated = lastCreatedUtc
if (this.stopLoading)
return
this.props.global.setLoading('Comparing comments to Reddit API...')
// All comments have been retrieved from Pushshift; wait for processing to finish
// Finished retrieving comments from Pushshift; wait for processing to finish
this.props.global.setLoading('Comparing comments to Reddit API...')
Promise.all(pushshiftPromises).then(lengths => {
console.log('Pushshift:', lengths.reduce((a,b) => a+b, 0), 'comments')
const pushshiftComments = lengths.reduce((a,b) => a+b, 0)
console.log('Pushshift:', pushshiftComments, 'comments')
// If Pushshift didn't find the Reddit commentHint, but should have, use Reddit's comment
if (commentHint && !pushshiftCommentLookup.has(commentHint.id) &&
commentHint.created_utc >= this.curContig().firstCreated && (
commentHint.created_utc < this.curContig().lastCreated || curContigLoadedAll
)) {
this.redditIdsToPushshift(commentHint)
pushshiftCommentLookup.set(commentHint.id, commentHint)
commentHint = undefined
}
// All comments from Pushshift have been processed; wait for Reddit to finish
while (!redditIdQueue.isEmpty())
doRedditComments(redditIdQueue.shiftChunk())
Promise.all(redditPromises).then(lengths => {
console.log('Reddit:', lengths.reduce((a,b) => a+b, 0), 'comments')
if (!this.stopLoading) {
this.props.global.setSuccess()
this.setState({
pushshiftCommentLookup,
removed: this.state.removed,
deleted: this.state.deleted,
loadedAllComments,
loadingComments: false,
reloadingComments: false
})
const loadedAllComments = Boolean(this.curContig().loadedAllComments)
if (persistent && !loadedAllComments && pushshiftComments <= newCommentCount - pushshiftChunkSize)
this.getComments(newCommentCount - pushshiftComments, true, commentHint)
else {
this.props.global.setSuccess()
this.setState({
pushshiftCommentLookup,
removed: this.state.removed,
deleted: this.state.deleted,
loadedAllComments,
loadingComments: false,
reloadingComments: false
})
}
}
})
})
})
.catch(e => this.props.global.setError(e, e.helpUrl))
.catch(e => {
this.props.global.setError(e, e.helpUrl)
if (this.curContig().lastCreated === undefined) {
this.contigs.splice(this.curContigIdx, 1)
if (this.curContigIdx >= this.contigs.length)
this.setCurContig(this.contigs.length - 1)
}
})
}
render () {
const { subreddit, id, author } = this.state.post
const { commentID } = this.props.match.params
const reloadingComments = this.state.reloadingComments || this.props.global.state.loadingMoreComments
const reloadingComments = this.state.loadingComments ||
this.state.reloadingComments ||
this.props.global.state.loadingMoreComments
const linkToRestOfComments = `/r/${subreddit}/comments/${id}/_/`
const isSingleComment = commentID !== undefined
@ -294,28 +473,32 @@ class Thread extends React.Component {
return (
<>
<Post {...this.state.post} />
<Post {...this.state.post} reloadingComments={reloadingComments} />
<CommentInfo
total={this.state.pushshiftCommentLookup.size}
removed={this.state.removed}
deleted={this.state.deleted}
/>
<SortBy
loadedAllComments={this.state.loadedAllComments}
reloadingComments={reloadingComments}
total={this.state.pushshiftCommentLookup.size}
/>
{
(!this.state.loadingComments && root) &&
<>
<CommentInfo
total={this.state.pushshiftCommentLookup.size}
removed={this.state.removed}
deleted={this.state.deleted}
/>
<SortBy
loadedAllComments={this.state.loadedAllComments}
reloadingComments={reloadingComments}
total={this.state.pushshiftCommentLookup.size}
/>
{isSingleComment &&
<div className='view-rest-of-comment'>
<div>you are viewing a single comment's thread.</div>
<Link to={linkToRestOfComments}>view the rest of the comments</Link>
{this.state.reloadingComments ?
<div className='faux-link'>view the rest of the comments</div> :
<Link to={linkToRestOfComments}>view the rest of the comments</Link>
}
</div>
}
<CommentSection
root={root}
postID={id}
comments={this.state.pushshiftCommentLookup}
postAuthor={isDeleted(author) ? null : author}
commentFilter={this.props.global.state.commentFilter} // need to explicitly

6
src/sass/comment.sass
View file

@ -57,6 +57,12 @@
margin-right: 4px
cursor: pointer
.comment-links a.wait
cursor: wait
.comment-links a.wait:hover
text-decoration: none
.comment-odd
background-color: #121212

5
src/sass/thread.sass
View file

@ -144,6 +144,9 @@
color: rgb(204, 204, 204)
margin-bottom: 10px
.faux-link
color: $link
.load-more
font-weight: bold
@ -156,7 +159,7 @@
cursor: pointer
.fade
animation: fade-anim 3s 2s ease-in-out forwards
animation: fade-anim 3s 5s ease-in-out forwards
@keyframes fade-anim
100%