mirror of
https://github.com/gurnec/removeddit.git
synced 2026-03-11 08:54:27 +00:00
Add context query param: shows permalink ancestors
Appending `?context=#` to a permalink will highlight the permalinked comment and display above it up to 8 ancestors, similar to Reddit's behaviour. To keep things fast, this update required that the Pushshift comments endpoint work with `ids` queries which was fixed around May 11.
This commit is contained in:
parent
32deb7f1e6
commit
85af5be925
8 changed files with 272 additions and 114 deletions
2
dist/main.css
vendored
2
dist/main.css
vendored
File diff suppressed because one or more lines are too long
|
|
@ -2,7 +2,9 @@ import { fetchJson, sleep } from '../../utils'
|
|||
|
||||
export const chunkSize = 100;
|
||||
const postURL = 'https://api.pushshift.io/reddit/submission/search/?fields=author,created_utc,domain,edited,id,link_flair_text,num_comments,permalink,position,removed_by_category,retrieved_on,retrieved_utc,score,selftext,subreddit,thumbnail,thumbnail_height,thumbnail_width,title,url&ids='
|
||||
const commentURL = `https://api.pushshift.io/reddit/comment/search/?metadata=true&size=${chunkSize}&sort=asc&fields=author,body,created_utc,id,link_id,parent_id,retrieved_on,retrieved_utc,score,subreddit&link_id=`
|
||||
const commentURL = 'https://api.pushshift.io/reddit/comment/search/?fields=author,body,created_utc,id,link_id,parent_id,retrieved_on,retrieved_utc,score,subreddit&'
|
||||
const commentURLbyIDs = `${commentURL}ids=`
|
||||
const commentURLbyLink = `${commentURL}metadata=true&size=${chunkSize}&sort=asc&link_id=`
|
||||
|
||||
const errorHandler = (msg, origError, from) => {
|
||||
console.error(from + ': ' + origError)
|
||||
|
|
@ -71,6 +73,41 @@ export const getPost = async threadID => {
|
|||
}
|
||||
}
|
||||
|
||||
// Starting sometime around May/6/2022, queries using the `ids` parameter
|
||||
// started returning decimal IDs for the `_id` members instead of fullnames.
|
||||
const toBase36 = id => {
|
||||
if (!id)
|
||||
return id
|
||||
if (typeof id == 'number')
|
||||
return id.toString(36)
|
||||
else
|
||||
return id[2] == '_' ? id.substring(3) : id
|
||||
}
|
||||
|
||||
export const getCommentsFromIds = async commentIDs => {
|
||||
if (commentIDs.length == 0)
|
||||
return []
|
||||
let response, delay = 0
|
||||
while (true) {
|
||||
await pushshiftTokenBucket.waitForToken()
|
||||
try {
|
||||
response = await fetchJson(`${commentURLbyIDs}${commentIDs.join()}`)
|
||||
break
|
||||
} catch (error) {
|
||||
if (delay >= 2000) // after ~4s of consecutive failures
|
||||
errorHandler('Could not get removed comments', error, 'pushshift.getCommentsFromIds') // rethrows
|
||||
delay = delay * 2 || 125
|
||||
pushshiftTokenBucket.setNextAvail(delay)
|
||||
console.log('pushshift.getCommentsFromIds delay: ' + delay)
|
||||
}
|
||||
}
|
||||
return response.data.map(c => {
|
||||
c.link_id = toBase36(c.link_id)
|
||||
c.parent_id = toBase36(c.parent_id) || c.link_id
|
||||
return c
|
||||
})
|
||||
}
|
||||
|
||||
// Comments w/a created_utc in the ranges below must be queried *without* the faster `q=*` parameter:
|
||||
// 1504224000 - 1506815999 (Sep/1/2017 0:00 - Sep/30/2017 23:59:59 UTC)
|
||||
// 1517443200 - 1522540799 (Feb/1/2018 0:00 - Mar/31/2018 23:59:59 UTC)
|
||||
|
|
@ -88,7 +125,7 @@ export const getComments = async (callback, threadID, maxComments, after = 0, be
|
|||
|
||||
let delay = 0
|
||||
while (true) {
|
||||
let query = commentURL + threadID
|
||||
let query = commentURLbyLink + threadID
|
||||
if (!inBrokenRange(after))
|
||||
query += '&q=*'
|
||||
if (after)
|
||||
|
|
|
|||
|
|
@ -43,3 +43,21 @@ export const getComments = commentIDs => (
|
|||
.then(results => results.data.children.map(({data}) => data))
|
||||
.catch(error => errorHandler(error, 'reddit.getComments'))
|
||||
)
|
||||
|
||||
// Fetch up to 8 of a comment's parents
|
||||
export const getParentComments = (threadID, commentID, parents) => {
|
||||
parents = Math.min(parents, 8)
|
||||
return fetchJson(
|
||||
`${baseURL}/comments/${threadID}?comment=${commentID}&context=${parents}&limit=${parents}&threaded=false&showmore=false`,
|
||||
requestSettings
|
||||
)
|
||||
.then(results => {
|
||||
const { children } = results[1].data
|
||||
// If there are fewer parents than requested, remove the comments which aren't parents
|
||||
const idx = children.findIndex(c => c.data.id == commentID)
|
||||
if (idx >= 0)
|
||||
children.splice(idx + 1)
|
||||
return children.map(({data}) => data)
|
||||
})
|
||||
.catch(error => errorHandler(error, 'reddit.getParentComments'))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@ const Comment = (props) => {
|
|||
} else {
|
||||
commentStyle += props.depth % 2 === 0 ? 'comment-even' : 'comment-odd'
|
||||
}
|
||||
if (props.id == props.highlightedID) {
|
||||
commentStyle += ' highlighted'
|
||||
}
|
||||
|
||||
let innerHTML, editedInnerHTML;
|
||||
if (props.removed && isRemoved(props.body)) {
|
||||
|
|
@ -46,7 +49,7 @@ const Comment = (props) => {
|
|||
>parent</NavLink>
|
||||
:
|
||||
// Use a function, not just an object--state is recreated and scrollBehavior is always initialized
|
||||
<Link to={() => ({hash: `#${props.parent_id}`, state: {scrollBehavior: 'smooth'}})}>parent</Link>
|
||||
<Link to={loc => ({...loc, hash: `#${props.parent_id}`, state: {scrollBehavior: 'smooth'}})}>parent</Link>
|
||||
)
|
||||
|
||||
return (
|
||||
|
|
@ -94,6 +97,7 @@ const Comment = (props) => {
|
|||
{...comment}
|
||||
depth={props.depth + 1}
|
||||
postAuthor={props.postAuthor}
|
||||
highlightedID={props.highlightedID}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import {
|
|||
showRemovedAndDeleted, showRemoved, showDeleted
|
||||
} from '../../utils'
|
||||
|
||||
const unflatten = (commentMap, rootID, postID) => {
|
||||
const unflatten = (commentMap, rootID, context, postID) => {
|
||||
const commentTree = []
|
||||
|
||||
commentMap.forEach(comment => {
|
||||
|
|
@ -58,6 +58,12 @@ const unflatten = (commentMap, rootID, postID) => {
|
|||
replies: missingRootReplies
|
||||
}
|
||||
}
|
||||
let newRoot
|
||||
while (context && rootComment.parent_id && (newRoot = commentMap.get(rootComment.parent_id))) {
|
||||
newRoot.replies = [rootComment]
|
||||
rootComment = newRoot
|
||||
context--
|
||||
}
|
||||
return [rootComment]
|
||||
}
|
||||
}
|
||||
|
|
@ -95,13 +101,13 @@ const filterCommentTree = (comments, filterFunction) => {
|
|||
return hasOkComment
|
||||
}
|
||||
|
||||
let commentTree, lastTotal, lastRoot, lastFilter, lastSort, lengthBeforeFiltering
|
||||
let commentTree, lastTotal, lastRoot, lastContext, lastFilter, lastSort, lengthBeforeFiltering
|
||||
|
||||
const commentSection = (props) => {
|
||||
console.time('Build comment tree')
|
||||
const {total, root, commentFilter, commentSort} = props
|
||||
const {total, root, context, commentFilter, commentSort} = props
|
||||
|
||||
const needsRebuild = !(total === lastTotal && root === lastRoot && (
|
||||
const needsRebuild = !(total === lastTotal && root === lastRoot && context === lastContext && (
|
||||
commentFilter === lastFilter ||
|
||||
lastFilter === filter.all ||
|
||||
lastFilter === filter.removedDeleted && (
|
||||
|
|
@ -110,7 +116,7 @@ const commentSection = (props) => {
|
|||
)
|
||||
))
|
||||
if (needsRebuild) {
|
||||
commentTree = unflatten(props.comments, root, props.postID)
|
||||
commentTree = unflatten(props.comments, root, context, props.postID)
|
||||
lengthBeforeFiltering = commentTree.length
|
||||
}
|
||||
|
||||
|
|
@ -142,6 +148,7 @@ const commentSection = (props) => {
|
|||
lastSort = commentSort
|
||||
console.timeEnd('Build comment tree')
|
||||
|
||||
props.setMoreContextAvail(commentTree.length > 0 && commentTree[0].parent_id != commentTree[0].link_id)
|
||||
props.setAllCommentsFiltered(commentTree.length == 0 && lengthBeforeFiltering > 0)
|
||||
|
||||
console.time('Build html tree')
|
||||
|
|
@ -153,6 +160,7 @@ const commentSection = (props) => {
|
|||
{...comment}
|
||||
depth={0}
|
||||
postAuthor={props.postAuthor}
|
||||
highlightedID={context ? root : null}
|
||||
/>
|
||||
))
|
||||
: <p>No comments found</p>
|
||||
|
|
@ -168,7 +176,8 @@ const areEqual = (prevProps, nextProps) => {
|
|||
return false
|
||||
if (nextProps.reloadingComments)
|
||||
return true
|
||||
return prevProps.total === nextProps.total &&
|
||||
return prevProps.total === nextProps.total &&
|
||||
prevProps.context === nextProps.context &&
|
||||
prevProps.postAuthor === nextProps.postAuthor
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import React from 'react'
|
||||
import {connect, maxCommentsDefault} from '../../state'
|
||||
|
||||
let loadingStarted = false, showLoadedCount = false, lastTotal, newComments
|
||||
let loadingStarted = false, showLoadedCount = false, lastTotal, lastContext, newComments
|
||||
const loadMoreComments = (props, maxComments) => {
|
||||
loadingStarted = true
|
||||
props.global.loadMoreComments(maxComments)
|
||||
|
|
@ -26,15 +26,22 @@ const loadMore = (props) => {
|
|||
if (maxCommentsPreferred >= maxCommentsDefault * 2)
|
||||
loadElements.push(<a key='pref' onClick={() => loadMoreComments(props, maxCommentsPreferred)}>load {maxCommentsPreferred} more comments</a>)
|
||||
}
|
||||
|
||||
// If loadMoreComments() was called and has completed (because reloadingComments is now false),
|
||||
if (loadingStarted) {
|
||||
showLoadedCount = true
|
||||
showLoadedCount = true // then showLoadedCount below
|
||||
newComments = props.total - lastTotal
|
||||
lastTotal = props.total
|
||||
lastContext = props.context
|
||||
loadingStarted = false
|
||||
} else if (lastTotal !== props.total) {
|
||||
showLoadedCount = false
|
||||
|
||||
// Otherwise if the comment total or context changed w/o calling loadMoreComments(),
|
||||
} else if (lastTotal !== props.total || lastContext !== props.context) {
|
||||
showLoadedCount = false // then don't showLoadedCount below
|
||||
lastTotal = props.total
|
||||
lastContext = props.context
|
||||
}
|
||||
|
||||
if (showLoadedCount) {
|
||||
loadElements.push(<span key='loaded' className='fade'>
|
||||
{newComments > 0 ? `loaded ${newComments} more comments` : 'no new comments found'}
|
||||
|
|
|
|||
|
|
@ -3,11 +3,13 @@ import { Link } from 'react-router-dom'
|
|||
import {
|
||||
getPost,
|
||||
getComments as getRedditComments,
|
||||
getParentComments,
|
||||
chunkSize as redditChunkSize
|
||||
} from '../../api/reddit'
|
||||
import {
|
||||
getPost as getPushshiftPost,
|
||||
getComments as getPushshiftComments,
|
||||
getCommentsFromIds,
|
||||
chunkSize as pushshiftChunkSize
|
||||
} from '../../api/pushshift'
|
||||
import { isDeleted, isRemoved, sleep } from '../../utils'
|
||||
|
|
@ -57,11 +59,14 @@ class Thread extends React.Component {
|
|||
pushshiftCommentLookup: new Map(),
|
||||
removed: 0,
|
||||
deleted: 0,
|
||||
context: 0,
|
||||
moreContextAvail: true,
|
||||
allCommentsFiltered: false,
|
||||
loadedAllComments: false,
|
||||
loadingComments: true,
|
||||
reloadingComments: false
|
||||
}
|
||||
nextMoreContextAvail = true
|
||||
nextAllCommentsFiltered = false
|
||||
|
||||
// A 'contig' is an object representing a contiguous block of comments currently being downloaded or already
|
||||
|
|
@ -212,8 +217,9 @@ class Thread extends React.Component {
|
|||
})
|
||||
|
||||
// The max_comments query parameter can increase the initial comments-to-download
|
||||
const maxComments = Math.max(this.props.global.maxComments, constrainMaxComments(
|
||||
parseInt((new URLSearchParams(this.props.location.search)).get('max_comments'))))
|
||||
const searchParams = new URLSearchParams(this.props.location.search)
|
||||
const maxComments = Math.max(this.props.global.maxComments,
|
||||
constrainMaxComments(parseInt(searchParams.get('max_comments'))))
|
||||
|
||||
// Get comments starting from the earliest available (not a permalink)
|
||||
if (commentID === undefined) {
|
||||
|
|
@ -234,6 +240,9 @@ class Thread extends React.Component {
|
|||
console.error('link_id mismatch:', comment)
|
||||
return
|
||||
}
|
||||
const context = parseInt(searchParams.get('context'))
|
||||
if (context > 0)
|
||||
this.contextPromise = this.getContext(context)
|
||||
this.contigs.unshift({firstCreated: comment?.created_utc || EARLIEST_CREATED})
|
||||
this.getComments(maxComments, false, comment)
|
||||
})
|
||||
|
|
@ -278,6 +287,7 @@ class Thread extends React.Component {
|
|||
|
||||
componentDidUpdate () {
|
||||
let { loadingComments } = this.state
|
||||
const { commentID } = this.props.match.params
|
||||
|
||||
// If the max-to-download Reload button or 'load more comments' was clicked
|
||||
const { loadingMoreComments } = this.props.global.state
|
||||
|
|
@ -289,71 +299,92 @@ class Thread extends React.Component {
|
|||
this.updateCurContig()
|
||||
this.getComments(loadingMoreComments, true)
|
||||
|
||||
// Otherwise if we're loading a comment tree we haven't downloaded yet
|
||||
} else if (!loadingComments && !this.state.reloadingComments && !this.updateCurContig()) {
|
||||
// Otherwise if we're not already downloading comments, check to see if we need to start
|
||||
} else if (!loadingComments && !this.state.reloadingComments) {
|
||||
|
||||
// If we haven't downloaded from the earliest available yet (not a permalink)
|
||||
const { commentID } = this.props.match.params
|
||||
if (commentID === undefined) {
|
||||
loadingComments = true
|
||||
this.setState({loadingComments})
|
||||
this.props.global.setLoading('Loading comments...')
|
||||
console.time('Load comments')
|
||||
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
|
||||
this.setCurContig(0)
|
||||
this.getComments(this.props.global.maxComments)
|
||||
// If we're loading a comment tree we haven't downloaded yet
|
||||
if (!this.updateCurContig()) {
|
||||
|
||||
// If we haven't downloaded this permalink yet
|
||||
} else if (!this.commentIdAttempts.has(commentID)) {
|
||||
this.commentIdAttempts.add(commentID)
|
||||
this.setState({reloadingComments: true})
|
||||
this.props.global.setLoading('Loading comments...')
|
||||
console.time('Load comments')
|
||||
let createdUtcNotFound // true if Reddit doesn't have the comment's created_utc
|
||||
getRedditComments([commentID])
|
||||
.then(([comment]) => {
|
||||
const created_utc = comment?.created_utc
|
||||
if (created_utc > EARLIEST_CREATED) {
|
||||
let insertBefore = this.contigs.findIndex(contig => created_utc < contig.firstCreated)
|
||||
if (insertBefore == -1)
|
||||
insertBefore = this.contigs.length
|
||||
// If we haven't downloaded from the earliest available yet (not a permalink)
|
||||
if (commentID === undefined) {
|
||||
loadingComments = true
|
||||
this.setState({loadingComments})
|
||||
this.props.global.setLoading('Loading comments...')
|
||||
console.time('Load comments')
|
||||
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
|
||||
this.setCurContig(0)
|
||||
this.getComments(this.props.global.maxComments)
|
||||
|
||||
// If comment isn't inside an existing contig, create a new one and start downloading
|
||||
if (insertBefore == 0 || created_utc >= this.contigs[insertBefore - 1].lastCreated) {
|
||||
this.contigs.splice(insertBefore, 0, {firstCreated: created_utc})
|
||||
this.setCurContig(insertBefore)
|
||||
this.fullnamesToShortIDs(comment)
|
||||
this.getComments(this.props.global.maxComments, false, comment)
|
||||
// Otherwise if we haven't downloaded this permalink yet
|
||||
} else if (!this.commentIdAttempts.has(commentID)) {
|
||||
this.commentIdAttempts.add(commentID)
|
||||
this.setState({reloadingComments: true})
|
||||
this.props.global.setLoading('Loading comments...')
|
||||
console.time('Load comments')
|
||||
let createdUtcNotFound // true if Reddit doesn't have the comment's created_utc
|
||||
getRedditComments([commentID])
|
||||
.then(([comment]) => {
|
||||
const created_utc = comment?.created_utc
|
||||
if (created_utc > EARLIEST_CREATED) {
|
||||
let insertBefore = this.contigs.findIndex(contig => created_utc < contig.firstCreated)
|
||||
if (insertBefore == -1)
|
||||
insertBefore = this.contigs.length
|
||||
|
||||
// Otherwise an earlier attempt to download it from Pushshift turned up nothing,
|
||||
} else {
|
||||
const { pushshiftCommentLookup } = this.state
|
||||
this.fullnamesToShortIDs(comment)
|
||||
this.useRedditComment(comment) // so use the Reddit comment instead
|
||||
this.setCurContig(insertBefore - 1) // (this was the failed earlier attempt)
|
||||
console.timeEnd('Load comments')
|
||||
this.props.global.setSuccess()
|
||||
this.setState({pushshiftCommentLookup, loadingComments: false, reloadingComments: false})
|
||||
// If comment isn't inside an existing contig, create a new one and start downloading
|
||||
if (insertBefore == 0 || created_utc >= this.contigs[insertBefore - 1].lastCreated) {
|
||||
this.contigs.splice(insertBefore, 0, {firstCreated: created_utc})
|
||||
this.setCurContig(insertBefore)
|
||||
this.fullnamesToShortIDs(comment)
|
||||
this.getComments(this.props.global.maxComments, false, comment)
|
||||
|
||||
// Otherwise an earlier attempt to download it from Pushshift turned up nothing,
|
||||
} else {
|
||||
this.fullnamesToShortIDs(comment)
|
||||
this.useRedditComment(comment) // so use the Reddit comment instead
|
||||
this.setCurContig(insertBefore - 1) // (this was the failed earlier attempt)
|
||||
console.timeEnd('Load comments')
|
||||
this.props.global.setSuccess()
|
||||
this.setState({loadingComments: false, reloadingComments: false})
|
||||
}
|
||||
} else
|
||||
createdUtcNotFound = true
|
||||
})
|
||||
.catch(() => createdUtcNotFound = true)
|
||||
.finally(() => {
|
||||
if (createdUtcNotFound) {
|
||||
// As a last resort, try to download starting from the previous contig;
|
||||
// this only occurs once per commentID due to the commentIdAttempts Set.
|
||||
if (this.curContigIdx > 0)
|
||||
this.setCurContig(this.curContigIdx - 1)
|
||||
// If there is no previous, create one
|
||||
else if (this.curContig().firstCreated != EARLIEST_CREATED)
|
||||
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
|
||||
this.getComments(this.props.global.maxComments)
|
||||
}
|
||||
} else
|
||||
createdUtcNotFound = true
|
||||
})
|
||||
.catch(() => createdUtcNotFound = true)
|
||||
.finally(() => {
|
||||
if (createdUtcNotFound) {
|
||||
// As a last resort, try to download starting from the previous contig;
|
||||
// this only occurs once per commentID due to the commentIdAttempts Set.
|
||||
if (this.curContigIdx > 0)
|
||||
this.setCurContig(this.curContigIdx - 1)
|
||||
// If there is no previous, create one
|
||||
else if (this.curContig().firstCreated != EARLIEST_CREATED)
|
||||
this.contigs.unshift({firstCreated: EARLIEST_CREATED})
|
||||
this.getComments(this.props.global.maxComments)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
} // end of "If we're loading a comment tree we haven't downloaded yet"
|
||||
|
||||
// Check if the context query parameter has changed
|
||||
if (commentID) {
|
||||
const context = parseInt((new URLSearchParams(this.props.location.search)).get('context')) || 0
|
||||
if (context > this.state.context) {
|
||||
this.setState({reloadingComments: true})
|
||||
this.props.global.setLoading('Loading comments...')
|
||||
console.time('Load comments')
|
||||
this.getContext(context) // also updates state.context
|
||||
.then(commentCount => {
|
||||
console.log('Reddit:', commentCount, 'comments')
|
||||
console.timeEnd('Load comments')
|
||||
this.props.global.setSuccess()
|
||||
this.setState({loadingComments: false, reloadingComments: false})
|
||||
})
|
||||
} else if (context >= 0 && context != this.state.context)
|
||||
this.setState({ context })
|
||||
}
|
||||
} // end of "If we're not already downloading comments, check to see if we need to start"
|
||||
|
||||
// Handle any requested scrolling
|
||||
const { location } = this.props
|
||||
if (location.state?.scrollBehavior && location.hash.length > 1 &&
|
||||
!loadingComments && !this.props.global.isErrored()) {
|
||||
|
|
@ -364,10 +395,54 @@ class Thread extends React.Component {
|
|||
}
|
||||
}
|
||||
|
||||
if (this.nextMoreContextAvail != this.state.moreContextAvail)
|
||||
this.setState({moreContextAvail: this.nextMoreContextAvail})
|
||||
if (this.nextAllCommentsFiltered != this.state.allCommentsFiltered)
|
||||
this.setState({allCommentsFiltered: this.nextAllCommentsFiltered})
|
||||
}
|
||||
|
||||
// Before calling, state.pushshiftCommentLookup must be populated.
|
||||
// Compares an array of Reddit comments with those in pushshiftCommentLookup,
|
||||
// updating them to reflect their removed/deleted/edited status and then
|
||||
// updates the state's removed/deleted count. Returns the number processed.
|
||||
compareAndUpdateComments (redditComments) {
|
||||
if (redditComments.length == 0)
|
||||
return 0
|
||||
const { pushshiftCommentLookup } = this.state
|
||||
let removed = 0, deleted = 0
|
||||
redditComments.forEach(redditComment => {
|
||||
let pushshiftComment = pushshiftCommentLookup.get(redditComment.id)
|
||||
if (pushshiftComment === undefined) {
|
||||
// When a parent comment is missing from pushshift, use the redditComment instead
|
||||
pushshiftComment = this.fullnamesToShortIDs(redditComment)
|
||||
pushshiftCommentLookup.set(pushshiftComment.id, pushshiftComment)
|
||||
} else {
|
||||
// Replace pushshift score with reddit (it's usually more accurate)
|
||||
pushshiftComment.score = redditComment.score
|
||||
}
|
||||
|
||||
// Check what is removed / deleted according to reddit
|
||||
if (isRemoved(redditComment.body)) {
|
||||
removed++
|
||||
pushshiftComment.removed = true
|
||||
} else if (isDeleted(redditComment.body)) {
|
||||
deleted++
|
||||
pushshiftComment.deleted = true
|
||||
} else if (pushshiftComment !== redditComment) {
|
||||
if (isRemoved(pushshiftComment.body)) {
|
||||
// If it's deleted in pushshift, but later restored by a mod, use the restored
|
||||
this.fullnamesToShortIDs(redditComment)
|
||||
pushshiftCommentLookup.set(redditComment.id, redditComment)
|
||||
} else if (pushshiftComment.body != redditComment.body) {
|
||||
pushshiftComment.edited_body = redditComment.body
|
||||
pushshiftComment.edited = redditComment.edited
|
||||
}
|
||||
}
|
||||
})
|
||||
this.setState({ removed: this.state.removed + removed, deleted: this.state.deleted + deleted })
|
||||
return redditComments.length
|
||||
}
|
||||
|
||||
// Before calling, either create (and set to current) a new contig to begin downloading
|
||||
// after a new time, or set the current contig to begin adding to the end of that contig.
|
||||
// persistent: if true, will try to continue downloading after the current contig has
|
||||
|
|
@ -410,40 +485,7 @@ class Thread extends React.Component {
|
|||
|
||||
// Download a list of comments by id from Reddit, and process them
|
||||
doRedditComments = ids => redditPromises.push(getRedditComments(ids)
|
||||
.then(comments => {
|
||||
let removed = 0, deleted = 0
|
||||
comments.forEach(comment => {
|
||||
let pushshiftComment = pushshiftCommentLookup.get(comment.id)
|
||||
if (pushshiftComment === undefined) {
|
||||
// When a parent comment is missing from pushshift, use the reddit comment instead
|
||||
pushshiftComment = this.redditIdsToPushshift(comment)
|
||||
pushshiftCommentLookup.set(comment.id, pushshiftComment)
|
||||
} else {
|
||||
// Replace pushshift score with reddit (it's usually more accurate)
|
||||
pushshiftComment.score = comment.score
|
||||
}
|
||||
|
||||
// Check what is removed / deleted according to reddit
|
||||
if (isRemoved(comment.body)) {
|
||||
removed++
|
||||
pushshiftComment.removed = true
|
||||
} else if (isDeleted(comment.body)) {
|
||||
deleted++
|
||||
pushshiftComment.deleted = true
|
||||
} else if (pushshiftComment !== comment) {
|
||||
if (isRemoved(pushshiftComment.body)) {
|
||||
// If it's deleted in pushshift, but later restored by a mod, use the restored
|
||||
this.redditIdsToPushshift(comment)
|
||||
pushshiftCommentLookup.set(comment.id, comment)
|
||||
} else if (pushshiftComment.body != comment.body) {
|
||||
pushshiftComment.edited_body = comment.body
|
||||
pushshiftComment.edited = comment.edited
|
||||
}
|
||||
}
|
||||
})
|
||||
this.setState({ removed: this.state.removed + removed, deleted: this.state.deleted + deleted })
|
||||
return comments.length
|
||||
})
|
||||
.then(comments => this.compareAndUpdateComments(comments))
|
||||
.catch(error => {
|
||||
console.timeEnd('Load comments')
|
||||
this.props.global.setError(error, error.helpUrl)
|
||||
|
|
@ -489,7 +531,10 @@ class Thread extends React.Component {
|
|||
// All comments from Pushshift have been processed; wait for Reddit to finish
|
||||
while (!redditIdQueue.isEmpty())
|
||||
doRedditComments(redditIdQueue.shiftChunk())
|
||||
if (this.contextPromise)
|
||||
redditPromises.push(this.contextPromise)
|
||||
Promise.all(redditPromises).then(lengths => {
|
||||
this.contextPromise = undefined
|
||||
console.log('Reddit:', lengths.reduce((a,b) => a+b, 0), 'comments')
|
||||
|
||||
if (!this.stopLoading) {
|
||||
|
|
@ -524,6 +569,26 @@ class Thread extends React.Component {
|
|||
})
|
||||
}
|
||||
|
||||
// Makes a best-effort attempt to retrieve context# ancestors of the current commentID.
|
||||
// Returns a Promise which resolves with the number retrieved, or rejects with undefined.
|
||||
getContext (context) {
|
||||
const { params } = this.props.match
|
||||
const { pushshiftCommentLookup } = this.state
|
||||
return getParentComments(params.threadID, params.commentID, context)
|
||||
.then(redditComments => {
|
||||
const ids = redditComments.map(c => c.id).filter(id => !pushshiftCommentLookup.has(id))
|
||||
return getCommentsFromIds(ids)
|
||||
.then(pushshiftComments => {
|
||||
if (ids.length)
|
||||
console.log('Pushshift:', pushshiftComments.length, 'comments')
|
||||
this.setState({ context })
|
||||
pushshiftComments.forEach(comment => pushshiftCommentLookup.set(comment.id, comment))
|
||||
return this.compareAndUpdateComments(redditComments)
|
||||
})
|
||||
})
|
||||
.catch(e => console.error(e))
|
||||
}
|
||||
|
||||
componentWillUnmount () {
|
||||
this.stopLoading = true
|
||||
}
|
||||
|
|
@ -557,19 +622,30 @@ class Thread extends React.Component {
|
|||
<>
|
||||
{isSingleComment &&
|
||||
<div className='view-rest-of-comment'>
|
||||
<div>you are viewing a single comment's thread.</div>
|
||||
<div>you are viewing a single comment's thread.</div><div>
|
||||
{this.state.reloadingComments ?
|
||||
<div className='faux-link'>view the rest of the comments</div> :
|
||||
<Link to={() => ({
|
||||
<span className='nowrap faux-link'>view the rest of the comments →</span> :
|
||||
<span className='nowrap'><Link to={() => ({
|
||||
pathname: `/r/${subreddit}/comments/${id}/_/`,
|
||||
hash: '#comment-info',
|
||||
state: {scrollBehavior: 'smooth'}}
|
||||
)}>view the rest of the comments</Link>
|
||||
)}>view the rest of the comments</Link> →</span>
|
||||
}
|
||||
</div>
|
||||
{this.state.moreContextAvail && this.state.context < 8 && <>
|
||||
<span className='space' />
|
||||
{this.state.reloadingComments ?
|
||||
<span className='nowrap faux-link'>view more context →</span> :
|
||||
<span className='nowrap'><Link to={() => ({
|
||||
pathname: `/r/${subreddit}/comments/${id}/_/${commentID}/`,
|
||||
search: `?context=${this.state.context < 4 ? 4 : 8}`}
|
||||
)}>view more context</Link> →</span>
|
||||
}
|
||||
</>}
|
||||
</div></div>
|
||||
}
|
||||
<CommentSection
|
||||
root={root}
|
||||
context={this.state.context}
|
||||
postID={id}
|
||||
comments={this.state.pushshiftCommentLookup}
|
||||
postAuthor={isDeleted(author) ? null : author}
|
||||
|
|
@ -577,12 +653,14 @@ class Thread extends React.Component {
|
|||
commentSort={this.props.global.state.commentSort} // pass in these props
|
||||
reloadingComments={reloadingComments} // to ensure React.memo
|
||||
total={this.state.pushshiftCommentLookup.size} // works correctly
|
||||
setMoreContextAvail={avail => this.nextMoreContextAvail = avail}
|
||||
setAllCommentsFiltered={filtered => this.nextAllCommentsFiltered = filtered}
|
||||
/>
|
||||
<LoadMore
|
||||
loadedAllComments={this.state.loadedAllComments}
|
||||
reloadingComments={reloadingComments}
|
||||
total={this.state.pushshiftCommentLookup.size}
|
||||
context={this.state.context}
|
||||
/>
|
||||
</>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@
|
|||
border-color: var(--l-comment-brd)
|
||||
color: var(--l-light)
|
||||
|
||||
.highlighted
|
||||
border: 2px solid var(--thread-rest-brd)
|
||||
@media (prefers-color-scheme: light)
|
||||
border-color: var(--l-thread-rest-brd)
|
||||
|
||||
.comment-head
|
||||
font-size: 10px
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue