Add context query param: shows permalink ancestors

Appending `?context=#` to a permalink will highlight the permalinked
comment and display above it up to 8 ancestors, similar to Reddit's
behaviour. To keep things fast, this update required that the Pushshift
comments endpoint work with `ids` queries which was fixed around May 11.
This commit is contained in:
Christopher Gurnee 2022-05-13 22:04:49 +00:00
parent 32deb7f1e6
commit 85af5be925
8 changed files with 272 additions and 114 deletions

2
dist/main.css vendored

File diff suppressed because one or more lines are too long

View file

@ -2,7 +2,9 @@ import { fetchJson, sleep } from '../../utils'
export const chunkSize = 100;
const postURL = 'https://api.pushshift.io/reddit/submission/search/?fields=author,created_utc,domain,edited,id,link_flair_text,num_comments,permalink,position,removed_by_category,retrieved_on,retrieved_utc,score,selftext,subreddit,thumbnail,thumbnail_height,thumbnail_width,title,url&ids='
const commentURL = `https://api.pushshift.io/reddit/comment/search/?metadata=true&size=${chunkSize}&sort=asc&fields=author,body,created_utc,id,link_id,parent_id,retrieved_on,retrieved_utc,score,subreddit&link_id=`
const commentURL = 'https://api.pushshift.io/reddit/comment/search/?fields=author,body,created_utc,id,link_id,parent_id,retrieved_on,retrieved_utc,score,subreddit&'
const commentURLbyIDs = `${commentURL}ids=`
const commentURLbyLink = `${commentURL}metadata=true&size=${chunkSize}&sort=asc&link_id=`
const errorHandler = (msg, origError, from) => {
console.error(from + ': ' + origError)
@ -71,6 +73,41 @@ export const getPost = async threadID => {
}
}
// Starting sometime around May/6/2022, queries using the `ids` parameter
// started returning decimal IDs for the `_id` members instead of fullnames.
const toBase36 = id => {
if (!id)
return id
if (typeof id == 'number')
return id.toString(36)
else
return id[2] == '_' ? id.substring(3) : id
}
export const getCommentsFromIds = async commentIDs => {
if (commentIDs.length == 0)
return []
let response, delay = 0
while (true) {
await pushshiftTokenBucket.waitForToken()
try {
response = await fetchJson(`${commentURLbyIDs}${commentIDs.join()}`)
break
} catch (error) {
if (delay >= 2000) // after ~4s of consecutive failures
errorHandler('Could not get removed comments', error, 'pushshift.getCommentsFromIds') // rethrows
delay = delay * 2 || 125
pushshiftTokenBucket.setNextAvail(delay)
console.log('pushshift.getCommentsFromIds delay: ' + delay)
}
}
return response.data.map(c => {
c.link_id = toBase36(c.link_id)
c.parent_id = toBase36(c.parent_id) || c.link_id
return c
})
}
// Comments w/a created_utc in the ranges below must be queried *without* the faster `q=*` parameter:
// 1504224000 - 1506815999 (Sep/1/2017 0:00 - Sep/30/2017 23:59:59 UTC)
// 1517443200 - 1522540799 (Feb/1/2018 0:00 - Mar/31/2018 23:59:59 UTC)
@ -88,7 +125,7 @@ export const getComments = async (callback, threadID, maxComments, after = 0, be
let delay = 0
while (true) {
let query = commentURL + threadID
let query = commentURLbyLink + threadID
if (!inBrokenRange(after))
query += '&q=*'
if (after)

View file

@ -43,3 +43,21 @@ export const getComments = commentIDs => (
.then(results => results.data.children.map(({data}) => data))
.catch(error => errorHandler(error, 'reddit.getComments'))
)
// Fetch up to 8 of a comment's parents
export const getParentComments = (threadID, commentID, parents) => {
parents = Math.min(parents, 8)
return fetchJson(
`${baseURL}/comments/${threadID}?comment=${commentID}&context=${parents}&limit=${parents}&threaded=false&showmore=false`,
requestSettings
)
.then(results => {
const { children } = results[1].data
// If there are fewer parents than requested, remove the comments which aren't parents
const idx = children.findIndex(c => c.data.id == commentID)
if (idx >= 0)
children.splice(idx + 1)
return children.map(({data}) => data)
})
.catch(error => errorHandler(error, 'reddit.getParentComments'))
}

View file

@ -12,6 +12,9 @@ const Comment = (props) => {
} else {
commentStyle += props.depth % 2 === 0 ? 'comment-even' : 'comment-odd'
}
if (props.id == props.highlightedID) {
commentStyle += ' highlighted'
}
let innerHTML, editedInnerHTML;
if (props.removed && isRemoved(props.body)) {
@ -46,7 +49,7 @@ const Comment = (props) => {
>parent</NavLink>
:
// Use a function, not just an object--state is recreated and scrollBehavior is always initialized
<Link to={() => ({hash: `#${props.parent_id}`, state: {scrollBehavior: 'smooth'}})}>parent</Link>
<Link to={loc => ({...loc, hash: `#${props.parent_id}`, state: {scrollBehavior: 'smooth'}})}>parent</Link>
)
return (
@ -94,6 +97,7 @@ const Comment = (props) => {
{...comment}
depth={props.depth + 1}
postAuthor={props.postAuthor}
highlightedID={props.highlightedID}
/>
))}
</div>

View file

@ -6,7 +6,7 @@ import {
showRemovedAndDeleted, showRemoved, showDeleted
} from '../../utils'
const unflatten = (commentMap, rootID, postID) => {
const unflatten = (commentMap, rootID, context, postID) => {
const commentTree = []
commentMap.forEach(comment => {
@ -58,6 +58,12 @@ const unflatten = (commentMap, rootID, postID) => {
replies: missingRootReplies
}
}
let newRoot
while (context && rootComment.parent_id && (newRoot = commentMap.get(rootComment.parent_id))) {
newRoot.replies = [rootComment]
rootComment = newRoot
context--
}
return [rootComment]
}
}
@ -95,13 +101,13 @@ const filterCommentTree = (comments, filterFunction) => {
return hasOkComment
}
let commentTree, lastTotal, lastRoot, lastFilter, lastSort, lengthBeforeFiltering
let commentTree, lastTotal, lastRoot, lastContext, lastFilter, lastSort, lengthBeforeFiltering
const commentSection = (props) => {
console.time('Build comment tree')
const {total, root, commentFilter, commentSort} = props
const {total, root, context, commentFilter, commentSort} = props
const needsRebuild = !(total === lastTotal && root === lastRoot && (
const needsRebuild = !(total === lastTotal && root === lastRoot && context === lastContext && (
commentFilter === lastFilter ||
lastFilter === filter.all ||
lastFilter === filter.removedDeleted && (
@ -110,7 +116,7 @@ const commentSection = (props) => {
)
))
if (needsRebuild) {
commentTree = unflatten(props.comments, root, props.postID)
commentTree = unflatten(props.comments, root, context, props.postID)
lengthBeforeFiltering = commentTree.length
}
@ -142,6 +148,7 @@ const commentSection = (props) => {
lastSort = commentSort
console.timeEnd('Build comment tree')
props.setMoreContextAvail(commentTree.length > 0 && commentTree[0].parent_id != commentTree[0].link_id)
props.setAllCommentsFiltered(commentTree.length == 0 && lengthBeforeFiltering > 0)
console.time('Build html tree')
@ -153,6 +160,7 @@ const commentSection = (props) => {
{...comment}
depth={0}
postAuthor={props.postAuthor}
highlightedID={context ? root : null}
/>
))
: <p>No comments found</p>
@ -168,7 +176,8 @@ const areEqual = (prevProps, nextProps) => {
return false
if (nextProps.reloadingComments)
return true
return prevProps.total === nextProps.total &&
return prevProps.total === nextProps.total &&
prevProps.context === nextProps.context &&
prevProps.postAuthor === nextProps.postAuthor
}

View file

@ -1,7 +1,7 @@
import React from 'react'
import {connect, maxCommentsDefault} from '../../state'
let loadingStarted = false, showLoadedCount = false, lastTotal, newComments
let loadingStarted = false, showLoadedCount = false, lastTotal, lastContext, newComments
const loadMoreComments = (props, maxComments) => {
loadingStarted = true
props.global.loadMoreComments(maxComments)
@ -26,15 +26,22 @@ const loadMore = (props) => {
if (maxCommentsPreferred >= maxCommentsDefault * 2)
loadElements.push(<a key='pref' onClick={() => loadMoreComments(props, maxCommentsPreferred)}>load {maxCommentsPreferred} more comments</a>)
}
// If loadMoreComments() was called and has completed (because reloadingComments is now false),
if (loadingStarted) {
showLoadedCount = true
showLoadedCount = true // then showLoadedCount below
newComments = props.total - lastTotal
lastTotal = props.total
lastContext = props.context
loadingStarted = false
} else if (lastTotal !== props.total) {
showLoadedCount = false
// Otherwise if the comment total or context changed w/o calling loadMoreComments(),
} else if (lastTotal !== props.total || lastContext !== props.context) {
showLoadedCount = false // then don't showLoadedCount below
lastTotal = props.total
lastContext = props.context
}
if (showLoadedCount) {
loadElements.push(<span key='loaded' className='fade'>
{newComments > 0 ? `loaded ${newComments} more comments` : 'no new comments found'}

View file

@ -3,11 +3,13 @@ import { Link } from 'react-router-dom'
import {
getPost,
getComments as getRedditComments,
getParentComments,
chunkSize as redditChunkSize
} from '../../api/reddit'
import {
getPost as getPushshiftPost,
getComments as getPushshiftComments,
getCommentsFromIds,
chunkSize as pushshiftChunkSize
} from '../../api/pushshift'
import { isDeleted, isRemoved, sleep } from '../../utils'
@ -57,11 +59,14 @@ class Thread extends React.Component {
pushshiftCommentLookup: new Map(),
removed: 0,
deleted: 0,
context: 0,
moreContextAvail: true,
allCommentsFiltered: false,
loadedAllComments: false,
loadingComments: true,
reloadingComments: false
}
nextMoreContextAvail = true
nextAllCommentsFiltered = false
// A 'contig' is an object representing a contiguous block of comments currently being downloaded or already
@ -212,8 +217,9 @@ class Thread extends React.Component {
})
// The max_comments query parameter can increase the initial comments-to-download
const maxComments = Math.max(this.props.global.maxComments, constrainMaxComments(
parseInt((new URLSearchParams(this.props.location.search)).get('max_comments'))))
const searchParams = new URLSearchParams(this.props.location.search)
const maxComments = Math.max(this.props.global.maxComments,
constrainMaxComments(parseInt(searchParams.get('max_comments'))))
// Get comments starting from the earliest available (not a permalink)
if (commentID === undefined) {
@ -234,6 +240,9 @@ class Thread extends React.Component {
console.error('link_id mismatch:', comment)
return
}
const context = parseInt(searchParams.get('context'))
if (context > 0)
this.contextPromise = this.getContext(context)
this.contigs.unshift({firstCreated: comment?.created_utc || EARLIEST_CREATED})
this.getComments(maxComments, false, comment)
})
@ -278,6 +287,7 @@ class Thread extends React.Component {
componentDidUpdate () {
let { loadingComments } = this.state
const { commentID } = this.props.match.params
// If the max-to-download Reload button or 'load more comments' was clicked
const { loadingMoreComments } = this.props.global.state
@ -289,71 +299,92 @@ class Thread extends React.Component {
this.updateCurContig()
this.getComments(loadingMoreComments, true)
// Otherwise if we're loading a comment tree we haven't downloaded yet
} else if (!loadingComments && !this.state.reloadingComments && !this.updateCurContig()) {
// Otherwise if we're not already downloading comments, check to see if we need to start
} else if (!loadingComments && !this.state.reloadingComments) {
// If we haven't downloaded from the earliest available yet (not a permalink)
const { commentID } = this.props.match.params
if (commentID === undefined) {
loadingComments = true
this.setState({loadingComments})
this.props.global.setLoading('Loading comments...')
console.time('Load comments')
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
this.setCurContig(0)
this.getComments(this.props.global.maxComments)
// If we're loading a comment tree we haven't downloaded yet
if (!this.updateCurContig()) {
// If we haven't downloaded this permalink yet
} else if (!this.commentIdAttempts.has(commentID)) {
this.commentIdAttempts.add(commentID)
this.setState({reloadingComments: true})
this.props.global.setLoading('Loading comments...')
console.time('Load comments')
let createdUtcNotFound // true if Reddit doesn't have the comment's created_utc
getRedditComments([commentID])
.then(([comment]) => {
const created_utc = comment?.created_utc
if (created_utc > EARLIEST_CREATED) {
let insertBefore = this.contigs.findIndex(contig => created_utc < contig.firstCreated)
if (insertBefore == -1)
insertBefore = this.contigs.length
// If we haven't downloaded from the earliest available yet (not a permalink)
if (commentID === undefined) {
loadingComments = true
this.setState({loadingComments})
this.props.global.setLoading('Loading comments...')
console.time('Load comments')
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
this.setCurContig(0)
this.getComments(this.props.global.maxComments)
// If comment isn't inside an existing contig, create a new one and start downloading
if (insertBefore == 0 || created_utc >= this.contigs[insertBefore - 1].lastCreated) {
this.contigs.splice(insertBefore, 0, {firstCreated: created_utc})
this.setCurContig(insertBefore)
this.fullnamesToShortIDs(comment)
this.getComments(this.props.global.maxComments, false, comment)
// Otherwise if we haven't downloaded this permalink yet
} else if (!this.commentIdAttempts.has(commentID)) {
this.commentIdAttempts.add(commentID)
this.setState({reloadingComments: true})
this.props.global.setLoading('Loading comments...')
console.time('Load comments')
let createdUtcNotFound // true if Reddit doesn't have the comment's created_utc
getRedditComments([commentID])
.then(([comment]) => {
const created_utc = comment?.created_utc
if (created_utc > EARLIEST_CREATED) {
let insertBefore = this.contigs.findIndex(contig => created_utc < contig.firstCreated)
if (insertBefore == -1)
insertBefore = this.contigs.length
// Otherwise an earlier attempt to download it from Pushshift turned up nothing,
} else {
const { pushshiftCommentLookup } = this.state
this.fullnamesToShortIDs(comment)
this.useRedditComment(comment) // so use the Reddit comment instead
this.setCurContig(insertBefore - 1) // (this was the failed earlier attempt)
console.timeEnd('Load comments')
this.props.global.setSuccess()
this.setState({pushshiftCommentLookup, loadingComments: false, reloadingComments: false})
// If comment isn't inside an existing contig, create a new one and start downloading
if (insertBefore == 0 || created_utc >= this.contigs[insertBefore - 1].lastCreated) {
this.contigs.splice(insertBefore, 0, {firstCreated: created_utc})
this.setCurContig(insertBefore)
this.fullnamesToShortIDs(comment)
this.getComments(this.props.global.maxComments, false, comment)
// Otherwise an earlier attempt to download it from Pushshift turned up nothing,
} else {
this.fullnamesToShortIDs(comment)
this.useRedditComment(comment) // so use the Reddit comment instead
this.setCurContig(insertBefore - 1) // (this was the failed earlier attempt)
console.timeEnd('Load comments')
this.props.global.setSuccess()
this.setState({loadingComments: false, reloadingComments: false})
}
} else
createdUtcNotFound = true
})
.catch(() => createdUtcNotFound = true)
.finally(() => {
if (createdUtcNotFound) {
// As a last resort, try to download starting from the previous contig;
// this only occurs once per commentID due to the commentIdAttempts Set.
if (this.curContigIdx > 0)
this.setCurContig(this.curContigIdx - 1)
// If there is no previous, create one
else if (this.curContig().firstCreated != EARLIEST_CREATED)
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
this.getComments(this.props.global.maxComments)
}
} else
createdUtcNotFound = true
})
.catch(() => createdUtcNotFound = true)
.finally(() => {
if (createdUtcNotFound) {
// As a last resort, try to download starting from the previous contig;
// this only occurs once per commentID due to the commentIdAttempts Set.
if (this.curContigIdx > 0)
this.setCurContig(this.curContigIdx - 1)
// If there is no previous, create one
else if (this.curContig().firstCreated != EARLIEST_CREATED)
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
this.getComments(this.props.global.maxComments)
}
})
}
}
})
}
} // end of "If we're loading a comment tree we haven't downloaded yet"
// Check if the context query parameter has changed
if (commentID) {
const context = parseInt((new URLSearchParams(this.props.location.search)).get('context')) || 0
if (context > this.state.context) {
this.setState({reloadingComments: true})
this.props.global.setLoading('Loading comments...')
console.time('Load comments')
this.getContext(context) // also updates state.context
.then(commentCount => {
console.log('Reddit:', commentCount, 'comments')
console.timeEnd('Load comments')
this.props.global.setSuccess()
this.setState({loadingComments: false, reloadingComments: false})
})
} else if (context >= 0 && context != this.state.context)
this.setState({ context })
}
} // end of "If we're not already downloading comments, check to see if we need to start"
// Handle any requested scrolling
const { location } = this.props
if (location.state?.scrollBehavior && location.hash.length > 1 &&
!loadingComments && !this.props.global.isErrored()) {
@ -364,10 +395,54 @@ class Thread extends React.Component {
}
}
if (this.nextMoreContextAvail != this.state.moreContextAvail)
this.setState({moreContextAvail: this.nextMoreContextAvail})
if (this.nextAllCommentsFiltered != this.state.allCommentsFiltered)
this.setState({allCommentsFiltered: this.nextAllCommentsFiltered})
}
// Before calling, state.pushshiftCommentLookup must be populated.
// Compares an array of Reddit comments with those in pushshiftCommentLookup,
// updating them to reflect their removed/deleted/edited status and then
// updates the state's removed/deleted count. Returns the number processed.
compareAndUpdateComments (redditComments) {
if (redditComments.length == 0)
return 0
const { pushshiftCommentLookup } = this.state
let removed = 0, deleted = 0
redditComments.forEach(redditComment => {
let pushshiftComment = pushshiftCommentLookup.get(redditComment.id)
if (pushshiftComment === undefined) {
// When a parent comment is missing from pushshift, use the redditComment instead
pushshiftComment = this.fullnamesToShortIDs(redditComment)
pushshiftCommentLookup.set(pushshiftComment.id, pushshiftComment)
} else {
// Replace pushshift score with reddit (it's usually more accurate)
pushshiftComment.score = redditComment.score
}
// Check what is removed / deleted according to reddit
if (isRemoved(redditComment.body)) {
removed++
pushshiftComment.removed = true
} else if (isDeleted(redditComment.body)) {
deleted++
pushshiftComment.deleted = true
} else if (pushshiftComment !== redditComment) {
if (isRemoved(pushshiftComment.body)) {
// If it's deleted in pushshift, but later restored by a mod, use the restored
this.fullnamesToShortIDs(redditComment)
pushshiftCommentLookup.set(redditComment.id, redditComment)
} else if (pushshiftComment.body != redditComment.body) {
pushshiftComment.edited_body = redditComment.body
pushshiftComment.edited = redditComment.edited
}
}
})
this.setState({ removed: this.state.removed + removed, deleted: this.state.deleted + deleted })
return redditComments.length
}
// Before calling, either create (and set to current) a new contig to begin downloading
// after a new time, or set the current contig to begin adding to the end of that contig.
// persistent: if true, will try to continue downloading after the current contig has
@ -410,40 +485,7 @@ class Thread extends React.Component {
// Download a list of comments by id from Reddit, and process them
doRedditComments = ids => redditPromises.push(getRedditComments(ids)
.then(comments => {
let removed = 0, deleted = 0
comments.forEach(comment => {
let pushshiftComment = pushshiftCommentLookup.get(comment.id)
if (pushshiftComment === undefined) {
// When a parent comment is missing from pushshift, use the reddit comment instead
pushshiftComment = this.redditIdsToPushshift(comment)
pushshiftCommentLookup.set(comment.id, pushshiftComment)
} else {
// Replace pushshift score with reddit (it's usually more accurate)
pushshiftComment.score = comment.score
}
// Check what is removed / deleted according to reddit
if (isRemoved(comment.body)) {
removed++
pushshiftComment.removed = true
} else if (isDeleted(comment.body)) {
deleted++
pushshiftComment.deleted = true
} else if (pushshiftComment !== comment) {
if (isRemoved(pushshiftComment.body)) {
// If it's deleted in pushshift, but later restored by a mod, use the restored
this.redditIdsToPushshift(comment)
pushshiftCommentLookup.set(comment.id, comment)
} else if (pushshiftComment.body != comment.body) {
pushshiftComment.edited_body = comment.body
pushshiftComment.edited = comment.edited
}
}
})
this.setState({ removed: this.state.removed + removed, deleted: this.state.deleted + deleted })
return comments.length
})
.then(comments => this.compareAndUpdateComments(comments))
.catch(error => {
console.timeEnd('Load comments')
this.props.global.setError(error, error.helpUrl)
@ -489,7 +531,10 @@ class Thread extends React.Component {
// All comments from Pushshift have been processed; wait for Reddit to finish
while (!redditIdQueue.isEmpty())
doRedditComments(redditIdQueue.shiftChunk())
if (this.contextPromise)
redditPromises.push(this.contextPromise)
Promise.all(redditPromises).then(lengths => {
this.contextPromise = undefined
console.log('Reddit:', lengths.reduce((a,b) => a+b, 0), 'comments')
if (!this.stopLoading) {
@ -524,6 +569,26 @@ class Thread extends React.Component {
})
}
// Makes a best-effort attempt to retrieve context# ancestors of the current commentID.
// Returns a Promise which resolves with the number retrieved, or rejects with undefined.
getContext (context) {
const { params } = this.props.match
const { pushshiftCommentLookup } = this.state
return getParentComments(params.threadID, params.commentID, context)
.then(redditComments => {
const ids = redditComments.map(c => c.id).filter(id => !pushshiftCommentLookup.has(id))
return getCommentsFromIds(ids)
.then(pushshiftComments => {
if (ids.length)
console.log('Pushshift:', pushshiftComments.length, 'comments')
this.setState({ context })
pushshiftComments.forEach(comment => pushshiftCommentLookup.set(comment.id, comment))
return this.compareAndUpdateComments(redditComments)
})
})
.catch(e => console.error(e))
}
componentWillUnmount () {
this.stopLoading = true
}
@ -557,19 +622,30 @@ class Thread extends React.Component {
<>
{isSingleComment &&
<div className='view-rest-of-comment'>
<div>you are viewing a single comment's thread.</div>
<div>you are viewing a single comment's thread.</div><div>
{this.state.reloadingComments ?
<div className='faux-link'>view the rest of the comments</div> :
<Link to={() => ({
<span className='nowrap faux-link'>view the rest of the comments &rarr;</span> :
<span className='nowrap'><Link to={() => ({
pathname: `/r/${subreddit}/comments/${id}/_/`,
hash: '#comment-info',
state: {scrollBehavior: 'smooth'}}
)}>view the rest of the comments</Link>
)}>view the rest of the comments</Link> &rarr;</span>
}
</div>
{this.state.moreContextAvail && this.state.context < 8 && <>
<span className='space' />
{this.state.reloadingComments ?
<span className='nowrap faux-link'>view more context &rarr;</span> :
<span className='nowrap'><Link to={() => ({
pathname: `/r/${subreddit}/comments/${id}/_/${commentID}/`,
search: `?context=${this.state.context < 4 ? 4 : 8}`}
)}>view more context</Link> &rarr;</span>
}
</>}
</div></div>
}
<CommentSection
root={root}
context={this.state.context}
postID={id}
comments={this.state.pushshiftCommentLookup}
postAuthor={isDeleted(author) ? null : author}
@ -577,12 +653,14 @@ class Thread extends React.Component {
commentSort={this.props.global.state.commentSort} // pass in these props
reloadingComments={reloadingComments} // to ensure React.memo
total={this.state.pushshiftCommentLookup.size} // works correctly
setMoreContextAvail={avail => this.nextMoreContextAvail = avail}
setAllCommentsFiltered={filtered => this.nextAllCommentsFiltered = filtered}
/>
<LoadMore
loadedAllComments={this.state.loadedAllComments}
reloadingComments={reloadingComments}
total={this.state.pushshiftCommentLookup.size}
context={this.state.context}
/>
</>
}

View file

@ -10,6 +10,11 @@
border-color: var(--l-comment-brd)
color: var(--l-light)
.highlighted
border: 2px solid var(--thread-rest-brd)
@media (prefers-color-scheme: light)
border-color: var(--l-thread-rest-brd)
.comment-head
font-size: 10px