From 4492064184872444abd1bb9afb96532bf3f20730 Mon Sep 17 00:00:00 2001 From: Jesper Wrang Date: Mon, 15 Jan 2018 12:02:04 +0100 Subject: [PATCH] Found a better way to parse threads from reddit --- src/js/pages/About.js | 4 +- src/js/pushshift/index.js | 1 + src/js/reddit/index.js | 137 +++++++++++++++++--------------------- src/js/reddit/token.js | 37 ++++++++++ src/js/utils/index.js | 8 ++- src/sass/about.sass | 6 -- src/sass/main.sass | 6 ++ 7 files changed, 114 insertions(+), 85 deletions(-) create mode 100644 src/js/reddit/token.js diff --git a/src/js/pages/About.js b/src/js/pages/About.js index 078dc05..3893bea 100644 --- a/src/js/pages/About.js +++ b/src/js/pages/About.js @@ -6,9 +6,9 @@ export default props => (

About

Display - removed + removed (by mods) and - deleted + deleted (by users) comments/threads from Reddit.

diff --git a/src/js/pushshift/index.js b/src/js/pushshift/index.js index f3619b4..24d8d9a 100644 --- a/src/js/pushshift/index.js +++ b/src/js/pushshift/index.js @@ -32,6 +32,7 @@ export const getThread = threadID => { ) } +// getComments (handle more than 100, try 50, 25 etc....) export const test = threadID => { const elasticQuery = { query: { diff --git a/src/js/reddit/index.js b/src/js/reddit/index.js index 13561ee..d0a78c5 100644 --- a/src/js/reddit/index.js +++ b/src/js/reddit/index.js @@ -1,61 +1,80 @@ -import { json } from 'utils' -import clientID from './clientID' +import { json, flatten } from 'utils' +import { fetchToken, redditAuth } from 'reddit/token' -// Reddit API - -// Headers for general api calls -const init = { - headers: { - Authorization: '', - }, -} - -let hasToken = false -let baseCommentTree = {} - -// Headers for getting reddit api token -const tokenInit = { - headers: { - Authorization: `Basic ${btoa(`${clientID}:`)}`, - 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8', - }, - method: 'POST', - body: `grant_type=${encodeURIComponent('https://oauth.reddit.com/grants/installed_client')}&device_id=DO_NOT_TRACK_THIS_DEVICE`, -} - -const fetchToken = () => { - if (hasToken) { - return Promise.resolve() - } - - return ( - fetch('https://www.reddit.com/api/v1/access_token', tokenInit) - .then(json) - .then(jsonData => { - init.headers.Authorization = `bearer ${jsonData.access_token}`; - hasToken = true - }) - ) -} +let cachedThread = {} +export const lookup = {} export const getThread = (subreddit, threadID) => ( fetchToken() - .then(() => fetch(`https://oauth.reddit.com/r/${subreddit}/comments/${threadID}`, init)) + .then(() => fetch(`https://oauth.reddit.com/r/${subreddit}/comments/${threadID}`, redditAuth)) .then(json) .then(results => { // Save the comments for later - baseCommentTree = results[1].data.children + cachedThread = results // Return the thread return results[0].data.children[0].data }) ) -export const getCommentIDs = () => { +const handleThreadComment = (comment, results) => { + if (comment.kind === 't1') { + // Has comment replies + if (comment.data.replies) { + // Handle all replies in the same way + comment.data.replies.data.children.forEach(reply => handleThreadComment(reply, results)) + delete comment.data.replies + } + + // Add to the return object + results.ids.push(comment.data.id) + // Store commment in lookup table + lookup[comment.data.id] = comment.data + } else if (comment.kind === 'more') { + if (comment.data.id === '_') { + // "continue this thread" comment + results.continueThisThreadIDs.push(comment.data.parent_id); + } else if (comment.data.children.length < comment.data.count) { + // "Load more"-comment (that is missing some of its children) + results.morechildrenIDs.push(comment.data.children) + } + + // Always add the "more"-comments children + results.ids.push(...comment.data.children) + } else { + console.error('WTF', comment.kind) + } + + return results +} + +const handleMoreChildren = () => { } -// HandleIDs.normal(thread); +const handleThread = thread => { + const comments = thread[1].data.children + + // Ugly "hack", using this object as a pointer for storing comments. + // Not very js-like + const results = { + ids: [], + continueThisThreadIDs: [], + morechildrenIDs: [], + } + + comments.map(comment => handleThreadComment(comment, results)) + const { ids, continueThisThreadIDs, morechildrenIDs } = results + + console.log(ids) + console.log(continueThisThreadIDs) + console.log(morechildrenIDs) + + handleMoreChildren() +} + +export const getCommentIDs = () => handleThread(cachedThread) + // return HandleIDs.morechildren() // .catch(function(error){ // return Promise.reject("Could not get comments from Reddit (moreChildren)"); @@ -261,40 +280,6 @@ export const getCommentIDs = () => { // }; // })(); - -// // ------------------------------------------------------------------------------ -// // ----------------------- Extract ID from comments ----------------------------- -// // ------------------------------------------------------------------------------ -// var Extract = (function(){ -// var normal = function(comment){ -// var data = comment.data; - -// if(comment.kind == "more") { // "Show more"-comment -// if(data.id === "_") { // = "continue this thread" comment -// Comments.countinuethread.push(data.parent_id); -// } else if(data.children.length < data.count){ // "Load more"-comment (that is missing some of its children) -// Comments.morechildren.push(data.children); -// } -// Comments.ids.push.apply(Comments.ids, data.children); -// } else { // Normal comment -// if(data.replies) { -// data.replies.data.children.forEach(function(child){ -// normal(child); -// }); -// delete data.replies; -// } - -// Comments.ids.push(data.id); -// Comments.lookup[data.id] = data; -// } -// }; - -// return { -// normal: normal -// }; -// })(); - - // // ------------------------------------------------------------------------------ // // ---------------------------- Generating HTML --------------------------------- // // ------------------------------------------------------------------------------ diff --git a/src/js/reddit/token.js b/src/js/reddit/token.js new file mode 100644 index 0000000..9193436 --- /dev/null +++ b/src/js/reddit/token.js @@ -0,0 +1,37 @@ +import { json } from 'utils' +import clientID from './clientID' + +// Connecting to Reddit API +let hasToken = false + +// Headers for general api calls +export const redditAuth = { + headers: { + Authorization: '', + }, +} + +// Headers for getting reddit api token +const tokenInit = { + headers: { + Authorization: `Basic ${btoa(`${clientID}:`)}`, + 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8', + }, + method: 'POST', + body: `grant_type=${encodeURIComponent('https://oauth.reddit.com/grants/installed_client')}&device_id=DO_NOT_TRACK_THIS_DEVICE`, +} + +export const fetchToken = () => { + if (hasToken) { + return Promise.resolve() + } + + return ( + fetch('https://www.reddit.com/api/v1/access_token', tokenInit) + .then(json) + .then(jsonData => { + redditAuth.headers.Authorization = `bearer ${jsonData.access_token}`; + hasToken = true + }) + ) +} diff --git a/src/js/utils/index.js b/src/js/utils/index.js index ed0aad9..e199291 100644 --- a/src/js/utils/index.js +++ b/src/js/utils/index.js @@ -2,6 +2,12 @@ import SnuOwnd from 'libraries/snuownd.js' const markdown = SnuOwnd.getParser() +// Flatten arrays one level +export const flatten = arr => arr.reduce( + (accumulator, value) => accumulator.concat(value), + [] +) + // Change bases export const toBase36 = number => parseInt(number, 10).toString(36) export const toBase10 = numberString => parseInt(numberString, 36) @@ -18,7 +24,7 @@ export const json = x => x.json() // Parse comments export const parse = text => markdown.render(text) -// UTC -> "Reddit time format" (e.g. 5 hours ago, just now, etc...) +// UTC to "Reddit time format" (e.g. 5 hours ago, just now, etc...) export const prettyDate = createdUTC => { const currentUTC = Math.floor((new Date()).getTime() / 1000) const secondDiff = currentUTC - createdUTC diff --git a/src/sass/about.sass b/src/sass/about.sass index da455de..af91164 100644 --- a/src/sass/about.sass +++ b/src/sass/about.sass @@ -7,12 +7,6 @@ max-width: 800px background-color: #161616 - .removed - color: $red - - .deleted - color: $blue - h2 margin: 20px 0px 12px diff --git a/src/sass/main.sass b/src/sass/main.sass index c7ee11f..c034abf 100644 --- a/src/sass/main.sass +++ b/src/sass/main.sass @@ -21,6 +21,12 @@ html, body .deleted background-color: $deleted +.removed-text + color: $red + +.deleted-text + color: $blue + a color: $link