prehistoric-systems/removeddit
1
0
Fork 0
mirror of https://github.com/gurnec/removeddit.git synced 2026-03-11 08:54:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Ensure all Pushshift comment reqs include q=*

Browse source 
Pushshift requests w/o `q=*` have been broken for about 2 weeks now.
Once they are working, this change will be reverted.
This commit is contained in:
Christopher Gurnee 2022-11-28 17:18:16 +00:00
parent bd6812d92d
commit 25063ac8e3
1 changed files with 9 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
src/api/pushshift/index.js
View file

@ -112,19 +112,20 @@ export const getCommentsFromIds = async commentIDs => {
// 1504224000 - 1506815999 (Sep/1/2017 0:00 - Sep/30/2017 23:59:59 UTC)
// 1517443200 - 1522540799 (Feb/1/2018 0:00 - Mar/31/2018 23:59:59 UTC)
// The ranges below subtract two weeks from the range start and optionally add two to the end.
const inBrokenRange = (utc, looseEnd = false) => looseEnd ?
utc > 1503014400 && utc < 1508025599 || utc > 1516233600 && utc < 1523750399 :
utc > 1503014400 && utc < 1506815999 || utc > 1516233600 && utc < 1522540799
// TODO: Pushshift requests w/o `q=*` are broken; revert this change when they're fixed
//const inBrokenRange = (utc, looseEnd = false) => looseEnd ?
// utc > 1503014400 && utc < 1508025599 || utc > 1516233600 && utc < 1523750399 :
// utc > 1503014400 && utc < 1506815999 || utc > 1516233600 && utc < 1522540799
// The callback() function is called with an Array of comments after each chunk is
// retrieved. It should return as quickly as possible (scheduling time-taking work
// later), and may return false to cause getComments to exit early, or true otherwise.
export const getComments = async (callback, threadID, maxComments, after = 0, before = undefined) => {
let chunks = Math.floor(maxComments / chunkSize), firstChunk = true, response, lastCreatedUtc = 1
let chunks = Math.floor(maxComments / chunkSize), response, lastCreatedUtc = 1
while (true) {
let query = commentURLbyLink + threadID
if (!inBrokenRange(after))
//if (!inBrokenRange(after))
query += '&q=*'
if (after)
query += `&after=${after}`
@ -154,9 +155,9 @@ export const getComments = async (callback, threadID, maxComments, after = 0, be
})))
// If there's a chance the comments are in a broken range, restart the retrieval
if (firstChunk && !after && (comments.length === 0 || inBrokenRange(comments[0].created_utc, true)))
return getComments(callback, threadID, maxComments, 1503014401, before)
firstChunk = false
//if (firstChunk && !after && (comments.length === 0 || inBrokenRange(comments[0].created_utc, true)))
// return getComments(callback, threadID, maxComments, 1503014401, before)
//firstChunk = false
const loadedAllComments = Object.prototype.hasOwnProperty.call(response.metadata, 'total_results') ?
response.metadata.results_returned >= response.metadata.total_results :