prehistoric-systems/removeddit
1
0
Fork 0
mirror of https://github.com/gurnec/removeddit.git synced 2026-03-11 08:54:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use Pushshift metadata to see if loadedAllComments

Browse source 
This saves one Pushshift request on about half of all (re-)loads
This commit is contained in:
Christopher Gurnee 2022-03-10 19:26:10 +00:00
parent 0081278e3c
commit 456b212d5f
1 changed files with 5 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/api/pushshift/index.js
View file

@ -2,7 +2,7 @@ import { fetchJson, sleep } from '../../utils'
export const chunkSize = 100;
const postURL = 'https://api.pushshift.io/reddit/submission/search/?ids='
const commentURL = `https://api.pushshift.io/reddit/comment/search/?size=${chunkSize}&sort=asc&fields=author,body,created_utc,id,link_id,parent_id,retrieved_on,retrieved_utc,score,subreddit&q=*&link_id=`
const commentURL = `https://api.pushshift.io/reddit/comment/search/?metadata=true&size=${chunkSize}&sort=asc&fields=author,body,created_utc,id,link_id,parent_id,retrieved_on,retrieved_utc,score,subreddit&q=*&link_id=`
const errorHandler = (msg, origError, from) => {
console.error(from + ': ' + origError)
@ -75,14 +75,14 @@ export const getPost = async threadID => {
// retrieved. It should return as quickly as possible (scheduling time-taking work
// later), and may return false to cause getComments to exit early, or true otherwise.
export const getComments = async (callback, threadID, maxComments, after) => {
let chunks = Math.floor(maxComments / chunkSize), comments, lastCreatedUtc = 1
let chunks = Math.floor(maxComments / chunkSize), response, lastCreatedUtc = 1
while (true) {
let delay = 0
while (true) {
await pushshiftTokenBucket.waitForToken()
try {
comments = (await fetchJson(`${commentURL}${threadID}${after ? `&after=${after}` : ''}`)).data
response = await fetchJson(`${commentURL}${threadID}${after ? `&after=${after}` : ''}`)
break
} catch (error) {
if (delay >= 8000) // after ~16s of consecutive failures
@ -92,13 +92,14 @@ export const getComments = async (callback, threadID, maxComments, after) => {
pushshiftTokenBucket.setNextAvail(delay)
}
}
const comments = response.data
const exitEarly = callback(comments.map(c => ({
...c,
parent_id: c.parent_id?.substring(3) || threadID,
link_id: c.link_id?.substring(3) || threadID
})))
const loadedAllComments = comments.length < chunkSize/2
const loadedAllComments = response.metadata.results_returned >= response.metadata.total_results
if (comments.length)
lastCreatedUtc = comments[comments.length - 1].created_utc
if (loadedAllComments || chunks <= 1 || exitEarly)