prehistoric-systems/removeddit
1
0
Fork 0
mirror of https://github.com/gurnec/removeddit.git synced 2026-03-11 08:54:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve error handling

Browse source
This commit is contained in:
Christopher Gurnee 2022-03-18 15:54:15 +00:00
parent ee1bef62dd
commit 2d436062ce
4 changed files with 117 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/api/pushshift/index.js
View file

@ -88,12 +88,14 @@ export const getComments = async (callback, threadID, maxComments, after) => {
if (delay >= 8000) // after ~16s of consecutive failures
errorHandler('Could not get removed comments', error, 'pushshift.getComments') // rethrows
delay = delay * 2 || 125
console.log('pushshift.getComments delay: ' + delay)
pushshiftTokenBucket.setNextAvail(delay)
if (!callback([]))
return [ lastCreatedUtc, false ]
console.log('pushshift.getComments delay: ' + delay)
}
}
const comments = response.data
const exitEarly = callback(comments.map(c => ({
const exitEarly = !callback(comments.map(c => ({
...c,
parent_id: c.parent_id?.substring(3) || threadID,
link_id: c.link_id?.substring(3) || threadID

1
src/api/reddit/index.js
View file

@ -7,6 +7,7 @@ const requestSettings = {headers: {"Accept-Language": "en"}}
const errorHandler = (origError, from) => {
console.error(from + ': ' + origError)
const error = new Error('Could not connect to Reddit')
error.origError = origError
if (origError.name == 'TypeError') { // The exception when blocked by Tracking Protection
// https://stackoverflow.com/a/9851769
const isFirefox = typeof InstallTrigger !== 'undefined'

24
src/pages/common/Post.js
View file

@ -4,7 +4,27 @@ import { prettyScore, prettyDate, prettyTimeDiff, exactDateTime, parse, redditTh
export default (props) => {
if (!props.title) {
return <div />
if (props.removed) {
const permalink = `/r/${props.subreddit}/comments/${props.id}`
return (
<div class="thread removed">
<div class="thread-score-box">
<div class="vote upvote" />
<div class="thread-score">?</div>
<div class="vote downvote" />
</div>
<a class="thumbnail thumbnail-default" href={permalink} />
<div class="thread-content">
<a class="thread-title" href={permalink}>[removed too quickly to be archived]</a>
<div class="total-comments">
<a href={`https://www.reddit.com${permalink}`}>reddit</a>&nbsp;
<a href={`https://reveddit.com${permalink}`}>reveddit</a>
</div>
</div>
</div>
)
} else
return <div />
}
const url = props.url.replace('https://www.reddit.com', '')
@ -41,7 +61,7 @@ export default (props) => {
const [showEdited, setShowEdited] = useState(false)
return (
<div className={`thread ${props.removed && 'removed'} ${props.deleted && 'deleted'}`}>
<div className={`thread ${props.removed ? 'removed' : props.deleted ? 'deleted' : ''}`}>
{props.position &&
<span className='post-rank'>{props.position}</span>}
<div className='thread-score-box'>

149
src/pages/thread/index.js
View file

@ -62,26 +62,33 @@ class Thread extends React.Component {
const { subreddit, threadID } = this.props.match.params
this.props.global.setLoading('Loading post...')
// Get post from reddit
// Get post from Reddit. Each code path below should end in either
// setLoading() on success (if comments are still loading), or
// setError() and assigning stopLoading = true on failure.
getPost(threadID)
.then(post => {
let edited_selftext
document.title = post.title
if (isDeleted(post.selftext))
post.deleted = true
else if (isRemoved(post.selftext) || post.removed_by_category)
post.removed = true
else if (post.edited) {
edited_selftext = post.selftext
post.selftext = '...' // temporarily remove it to avoid flashing it onscreen
}
this.setState({ post })
if (!post.deleted && !post.removed && !post.edited) {
this.setState({ post })
if (this.state.loadingComments)
this.props.global.setLoading('Loading comments from Pushshift...')
// Fetch the post from Pushshift if it was deleted/removed/edited
if (post.deleted || post.removed || post.edited) {
} else {
const redditSelftext = post.selftext
post.selftext = '...' // temporarily remove selftext to avoid flashing it onscreen
this.setState({ post })
getPushshiftPost(threadID)
.then(origPost => {
if (origPost) {
if (post.deleted || post.removed) { // use the post from Pushshift instead
// If found on Pushshift, and deleted on Reddit, use Pushshift's post instead
if (post.deleted || post.removed) {
origPost.score = post.score
origPost.num_comments = post.num_comments
origPost.edited = post.edited
@ -90,44 +97,66 @@ class Thread extends React.Component {
else
origPost.removed = true
this.setState({ post: origPost })
} else { // it was only edited - update (if necessary) and use the Reddit post
if (edited_selftext != origPost.selftext && !isRemoved(origPost.selftext)) {
// If found on Pushshift, but it was only edited, update and use the Reddit post
} else {
if (redditSelftext != origPost.selftext && !isRemoved(origPost.selftext)) {
post.selftext = origPost.selftext
post.edited_selftext = edited_selftext
}
post.edited_selftext = redditSelftext
} else
post.selftext = redditSelftext // edited selftext not archived by Pushshift, use Reddit's
this.setState({ post })
}
} else if (post.edited) {
post.selftext = edited_selftext // restore it (after temporarily removing it above)
// Else if not found on Pushshift, nothing to do except restore the selftext (removed above)
} else {
post.selftext = redditSelftext
this.setState({ post })
}
if (this.state.loadingComments)
this.props.global.setLoading('Loading comments from Pushshift...')
})
.catch(e => {
this.props.global.setError(e, e.helpUrl)
if (post.edited) {
post.selftext = edited_selftext
this.setState({ post })
}
.catch(error => {
this.props.global.setError(error, error.helpUrl)
this.stopLoading = true
post.selftext = redditSelftext // restore it (after temporarily removing it above)
this.setState({ post })
})
}
})
.catch(error => {
this.props.global.setError(error)
// Fetch the post from pushshift on other errors (e.g. posts from banned subreddits)
getPushshiftPost(threadID)
.then(removedPost => {
document.title = removedPost.title
this.setState({ post: { ...removedPost, removed: true } })
})
.catch(error => {
this.props.global.setError(error, error.helpUrl)
// Create a dummy post so that comments will still be displayed
this.setState({ post: { subreddit, id: threadID } })
})
})
.finally(() => {
if (this.state.loadingComments)
this.props.global.setLoading('Loading comments from Pushshift...')
const origMessage = error.origError?.message
// Fetch the post from Pushshift if quarantined/banned (403) or not found (404)
if (origMessage && (origMessage.startsWith('403') || origMessage.startsWith('404'))) {
getPushshiftPost(threadID)
.then(removedPost => {
if (removedPost) {
document.title = removedPost.title
this.setState({ post: { ...removedPost, removed: true } })
if (this.state.loadingComments)
this.props.global.setLoading('Loading comments from Pushshift...')
} else {
if (origMessage.startsWith('403')) { // If Reddit admits it exists but Pushshift can't find it, then
this.setState({ post: { id: threadID, subreddit, removed: true } }) // create a dummy post and continue
if (this.state.loadingComments)
this.props.global.setLoading('Loading comments from Pushshift...')
} else {
this.props.global.setError({ message: '404 Post not found' })
this.stopLoading = true
}
}
})
.catch(error => {
this.props.global.setError(error, error.helpUrl)
this.stopLoading = true
})
} else {
this.props.global.setError(error, error.helpUrl)
this.stopLoading = true
}
})
const maxCommentsQuery = constrainMaxComments(
@ -150,29 +179,31 @@ class Thread extends React.Component {
const { pushshiftCommentLookup } = this.state
const redditIdQueue = new ChunkedQueue(redditChunkSize)
const pushshiftPromises = [], redditPromises = []
let redditError = false, doRedditComments
let doRedditComments
// Process a chunk of comments downloaded from Pushshift (started below)
const processPushshiftComments = comments => {
pushshiftPromises.push(sleep(0).then(() => {
let count = 0
comments.forEach(comment => {
const { id, parent_id } = comment
if (!pushshiftCommentLookup.has(id)) {
pushshiftCommentLookup.set(id, comment)
redditIdQueue.push(id)
count++
if (parent_id != threadID && !pushshiftCommentLookup.has(parent_id)) {
pushshiftCommentLookup.set(parent_id, undefined)
redditIdQueue.push(parent_id)
if (comments.length && !this.stopLoading) {
pushshiftPromises.push(sleep(0).then(() => {
let count = 0
comments.forEach(comment => {
const { id, parent_id } = comment
if (!pushshiftCommentLookup.has(id)) {
pushshiftCommentLookup.set(id, comment)
redditIdQueue.push(id)
count++
if (parent_id != threadID && !pushshiftCommentLookup.has(parent_id)) {
pushshiftCommentLookup.set(parent_id, undefined)
redditIdQueue.push(parent_id)
}
}
}
})
while (redditIdQueue.hasFullChunk())
doRedditComments(redditIdQueue.shiftChunk())
return count
}))
return redditError // causes getPushshiftComments() to exit early on a Reddit error
})
while (redditIdQueue.hasFullChunk())
doRedditComments(redditIdQueue.shiftChunk())
return count
}))
}
return !this.stopLoading // causes getPushshiftComments() to exit early if set
}
// Download a list of comments by id from Reddit, and process them
@ -214,7 +245,7 @@ class Thread extends React.Component {
})
.catch(error => {
this.props.global.setError(error, error.helpUrl)
redditError = true
this.stopLoading = true
})
)
@ -222,7 +253,7 @@ class Thread extends React.Component {
getPushshiftComments(processPushshiftComments, threadID, newCommentCount, after)
.then(([lastCreatedUtc, loadedAllComments]) => {
this.lastCreatedUtc = lastCreatedUtc
if (redditError)
if (this.stopLoading)
return
this.props.global.setLoading('Comparing comments to Reddit API...')
@ -235,7 +266,7 @@ class Thread extends React.Component {
doRedditComments(redditIdQueue.shiftChunk())
Promise.all(redditPromises).then(lengths => {
console.log('Reddit:', lengths.reduce((a,b) => a+b, 0), 'comments')
if (!redditError) {
if (!this.stopLoading) {
this.props.global.setSuccess()
this.setState({
pushshiftCommentLookup,