diff --git a/.eslintrc.json b/.eslintrc similarity index 87% rename from .eslintrc.json rename to .eslintrc index 5e36fe5..4b9d783 100644 --- a/.eslintrc.json +++ b/.eslintrc @@ -38,7 +38,10 @@ ] } ], - "jsx-quotes": "off", + "jsx-quotes": [ + "error", + "prefer-single" + ], "max-len": "warn", "no-param-reassign": "warn", "no-plusplus": [ @@ -54,6 +57,11 @@ "react/jsx-filename-extension": "off", "react/no-danger": "off", "react/prop-types": "off", - "semi": "off" + "semi": [ + "error", + "never" + ], + "no-use-before-define": "off", + "import/first": "off" } } diff --git a/LICENSE b/LICENSE index 7325d22..97e1d4b 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018-preset, Removeddit +Copyright (c) 2018-preset, Jesper Wrang Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/package-lock.json b/package-lock.json index a99f3f9..acb770c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9081,6 +9081,9 @@ "hoek": "2.16.3" } }, + "snuownd": { + "version": "git+https://github.com/JordanMilne/snuownd.git#06a1abc16ad06a1a6eff14afb7f9bcdb34e8ad01" + }, "sockjs": { "version": "0.3.19", "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz", diff --git a/package.json b/package.json index e0c2969..10355a4 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "react-redux": "^5.0.6", "react-router-dom": "^4.2.2", "redux": "^3.7.2", + "snuownd": "git+https://github.com/JordanMilne/snuownd.git", "whatwg-fetch": "^2.0.3" }, "devDependencies": { diff --git a/src/js/pushshift/index.js b/src/js/api/pushshift/index.js similarity index 100% rename from src/js/pushshift/index.js rename to src/js/api/pushshift/index.js diff --git a/src/js/api/reddit/auth.js b/src/js/api/reddit/auth.js new file mode 100644 index 0000000..ab825c2 --- /dev/null +++ b/src/js/api/reddit/auth.js @@ -0,0 +1,11 @@ +import { getToken } from './token' + +// Header for general api calls +export const getAuth = () => ( + getToken() + .then(token => ({ + headers: { + Authorization: `bearer ${token}`, + }, + })) +) diff --git a/src/js/reddit/clientID.js b/src/js/api/reddit/clientID.js similarity index 100% rename from src/js/reddit/clientID.js rename to src/js/api/reddit/clientID.js diff --git a/src/js/api/reddit/comment.js b/src/js/api/reddit/comment.js new file mode 100644 index 0000000..4842b87 --- /dev/null +++ b/src/js/api/reddit/comment.js @@ -0,0 +1,100 @@ +import { json, fetchMultiple, jsonMultiple, unique } from 'utils' +import { getAuth } from './auth' +import { getThread } from './thread' + +export const commentLookup = {} + +export const getCommentIDs = ( + subreddit, + threadID, + commentID = '', + results = { // This is ugly i know... + ids: [], + continueThisThreadIDs: {}, + morechildrenIDs: {}, + } +) => ( + getComments(subreddit, threadID, commentID) + .then(comments => handleComments(comments, subreddit, threadID, commentID, results)) + .then(() => debug(results)) + .then(() => results.ids) +) + +const getComments = (subreddit, threadID, commentID) => ( + getThread(subreddit, threadID, commentID) + .then(thread => thread[1].data.children) +) + +const handleComments = (comments, subreddit, threadID, commentID, results) => { + // Init ID arrays for this thread of comments () + results.continueThisThreadIDs[commentID] = [] + results.morechildrenIDs[commentID] = [] + + comments.forEach(comment => handleComment(comment, commentID, results)) + + return ( + handleMoreChildren(threadID, commentID, results) + .then(() => ( + Promise.all(results.continueThisThreadIDs[commentID].map(id => + getCommentIDs(subreddit, threadID, id, results))) + )) + ) +} + +const handleComment = (comment, commentID, results) => { + // Normal comment + if (comment.kind === 't1') { + // Has comment replies + if (comment.data.replies) { + // Handle all replies in the same way + comment.data.replies.data.children.forEach(commentReply => handleComment(commentReply, commentID, results)) + delete comment.data.replies + } + + // Add to the return object + results.ids.push(comment.data.id) + // Store commment in lookup table + commentLookup[comment.data.id] = comment.data + + // Special comment + } else if (comment.kind === 'more') { + // "continue this thread" comment + if (comment.data.id === '_') { + results.continueThisThreadIDs[commentID].push(comment.data.parent_id.split('_')[1]) + + // "Load more"-comment (that is missing some of its children) + } else if (comment.data.children.length < comment.data.count) { + results.morechildrenIDs[commentID].push(comment.data.children) + } + + // Always add the "more"-comments children + results.ids.push(...comment.data.children) + } +} + +const handleMoreChildren = (threadID, commentID, results) => ( + getAuth() + .then(auth => ( + Promise.all(results.morechildrenIDs[commentID].map(idArray => + fetchMultiple(`https://oauth.reddit.com/api/morechildren?link_id=t3_${threadID}&children=`, idArray, auth))) + .then(responsesArrays => Promise.all(responsesArrays.map(jsonMultiple))) + .then(responsesArrays => { + // Reset the array and parse for new morechildren-comments + results.morechildrenIDs[commentID] = [] + responsesArrays.forEach(responses => responses.forEach(response => response.jquery[10][3][0].forEach(comment => { + handleComment(comment, commentID, results) + }))) + }) + .then(() => { + if (results.morechildrenIDs[commentID].length !== 0) { + return handleMoreChildren(threadID, commentID, results) + } + + return Promise.resolve() + }) + )) +) + +const debug = results => { + console.log(JSON.parse(JSON.stringify(results))) +} diff --git a/src/js/reddit/index.js b/src/js/api/reddit/index.js similarity index 73% rename from src/js/reddit/index.js rename to src/js/api/reddit/index.js index e195096..e9a44a1 100644 --- a/src/js/reddit/index.js +++ b/src/js/api/reddit/index.js @@ -1,25 +1,5 @@ -export { getThread, extractPost } from './thread' +export { getPost } from './thread' export { getCommentIDs } from './comment' - -// return HandleIDs.morechildren() -// .catch(function(error){ -// return Promise.reject("Could not get comments from Reddit (moreChildren)"); -// }); -// }) -// .then(function(){ -// return Promise.all(_.map(_.uniq(Comments.countinuethread), function(id) { -// return fetch(URLs.thread+"/_/"+id.split("_")[1], Reddit.init) -// .then(Fetch2.json) -// .catch(function(error){ return Promise.reject("Could not get comments from Reddit (continueThisThread)") }) -// })) -// .catch(function(error){ -// return Promise.reject("Could not get comments from Reddit (continueThisThread)"); -// }); -// }) -// .then(function(smallerThreads){ -// _.forEach(smallerThreads, function(thread){ -// HandleIDs.normal(thread); -// }) // Status.loading("Getting removed comments..."); // HandleIDs.removed(); // ThreadHTML.createCommentInfo(Comments.removed.length); @@ -50,7 +30,6 @@ export { getCommentIDs } from './comment' // // ------------------------------------------------------------------------------ // var Comments = (function() { // var totalComments; -// var lookup = {}; // var toBeCreated = []; // var getParentComments = function(toLookup){ @@ -100,16 +79,10 @@ export { getCommentIDs } from './comment' // }; -// return { -// ids: [], // The comments we found -// morechildren: [], -// countinuethread: [], - // allIDs: [], // All the comments that we were suppose to find // removed: [], // deleted: [], // toBeCreated: toBeCreated, -// lookup: lookup, // getTotalComments: function() { return totalComments; }, // setTotalComments: function(total) { totalComments = total; }, @@ -151,30 +124,6 @@ export { getCommentIDs } from './comment' // } // }})(); - -// // ------------------------------------------------------------------------------ -// // ----------------- Handle comments from different requests -------------------- -// // ------------------------------------------------------------------------------ -// var morechildren = function(){ -// return Promise.all(_.map(_.uniq(Comments.morechildren), function(idArray){ -// return Fetch2.multiple(URLs.format(URLs.moreChildren, idArray), Reddit.init); -// })) -// .then(function(responseArrays){ -// Comments.morechildren.length = 0; -// _.forEach(responseArrays, function(responseArray){ -// _.forEach(responseArray, function(response){ -// _.forEach(response.jquery[10][3][0], function(comment){ -// Extract.normal(comment); -// }) -// }); -// }); -// }).then(function(){ -// if(Comments.morechildren.length !== 0) { -// return morechildren(); -// } -// }); -// }; - // var removed = function(){ // Comments.removed = _.difference(Comments.allIDs, Comments.ids); @@ -194,12 +143,6 @@ export { getCommentIDs } from './comment' // Comments.removed = _.uniq(Comments.removed); // }; -// return { -// normal: normal, -// morechildren: morechildren, -// removed: removed -// }; -// })(); // // ------------------------------------------------------------------------------ // // ---------------------------- Generating HTML --------------------------------- diff --git a/src/js/reddit/thread.js b/src/js/api/reddit/thread.js similarity index 51% rename from src/js/reddit/thread.js rename to src/js/api/reddit/thread.js index 0958b8f..421b001 100644 --- a/src/js/reddit/thread.js +++ b/src/js/api/reddit/thread.js @@ -1,36 +1,40 @@ import { json } from 'utils' -import { fetchToken, redditAuth } from 'reddit/token' +import { getAuth } from './auth' const cachedThreads = {} -export const getThread = (subreddit, threadID, continueThisThreadID = '') => { +// Thread = Post + Comments +// Return the post itself +export const getPost = (subreddit, threadID) => ( + getThread(subreddit, threadID) + .then(thread => thread[0].data.children[0].data) +) + +export const getThread = (subreddit, threadID, commentID = '') => { // We have already downloaded the thread and can use a cached copy if (cachedThreads.hasOwnProperty(threadID)) { - if (cachedThreads[threadID].hasOwnProperty(continueThisThreadID)) { - return Promise.resolve(cachedThreads[threadID][continueThisThreadID]) + if (cachedThreads[threadID].hasOwnProperty(commentID)) { + return Promise.resolve(cachedThreads[threadID][commentID]) } } - const url = `https://oauth.reddit.com/r/${subreddit}/comments/${threadID}/_/${continueThisThreadID}` + const url = `https://oauth.reddit.com/r/${subreddit}/comments/${threadID}/_/${commentID}` // Fetch thread from reddit return ( - fetchToken() - .then(() => fetch(url, redditAuth)) + getAuth() + .then(auth => fetch(url, auth)) .then(json) .then(thread => { - // Save the thread for later + // Create cache object for thread if it doesn't exists if (!cachedThreads.hasOwnProperty(threadID)) { cachedThreads[threadID] = {} } - cachedThreads[threadID][continueThisThreadID] = thread + // Save the thread for later + cachedThreads[threadID][commentID] = thread // Return the thread return thread }) ) } - -// Thread = Post + Comments -// Return the post itself -export const extractPost = thread => thread[0].data.children[0].data diff --git a/src/js/reddit/token.js b/src/js/api/reddit/token.js similarity index 59% rename from src/js/reddit/token.js rename to src/js/api/reddit/token.js index 9193436..86385bd 100644 --- a/src/js/reddit/token.js +++ b/src/js/api/reddit/token.js @@ -1,15 +1,8 @@ import { json } from 'utils' import clientID from './clientID' -// Connecting to Reddit API -let hasToken = false - -// Headers for general api calls -export const redditAuth = { - headers: { - Authorization: '', - }, -} +// Token for reddit API +let token = null // Headers for getting reddit api token const tokenInit = { @@ -21,17 +14,17 @@ const tokenInit = { body: `grant_type=${encodeURIComponent('https://oauth.reddit.com/grants/installed_client')}&device_id=DO_NOT_TRACK_THIS_DEVICE`, } -export const fetchToken = () => { - if (hasToken) { - return Promise.resolve() +export const getToken = () => { + if (token !== null) { + return Promise.resolve(token) } return ( fetch('https://www.reddit.com/api/v1/access_token', tokenInit) .then(json) - .then(jsonData => { - redditAuth.headers.Authorization = `bearer ${jsonData.access_token}`; - hasToken = true + .then(response => { + token = response.access_token + return token }) ) } diff --git a/src/js/api/removeddit/index.js b/src/js/api/removeddit/index.js new file mode 100644 index 0000000..e69de29 diff --git a/src/js/index.js b/src/js/index.js index 7a71d0f..430e25f 100644 --- a/src/js/index.js +++ b/src/js/index.js @@ -1,17 +1,23 @@ -import React from 'react' -import ReactDOM from 'react-dom' -import { Provider } from 'react-redux' -import { store } from 'state' -import App from 'App' -// import { setStatusLoading } from 'state' +// import React from 'react' +// import ReactDOM from 'react-dom' +// import { Provider } from 'react-redux' +// import { store } from 'state' +// import App from 'App' +// // import { setStatusLoading } from 'state' -import '../sass/main.sass' +// import '../sass/main.sass' -ReactDOM.render( - - - , - document.getElementById('app') -) +// ReactDOM.render( +// +// +// , +// document.getElementById('app') +// ) // store.dispatch(setStatusLoading('hi')) + +import { getCommentIDs } from 'api/reddit' +import { unique } from 'utils' + +getCommentIDs('TwoXChromosomes', '6z1hch') + .then(ids => console.log(unique(ids))) diff --git a/src/js/pages/Thread.js b/src/js/pages/Thread.js index 5a32347..b2615b6 100644 --- a/src/js/pages/Thread.js +++ b/src/js/pages/Thread.js @@ -28,23 +28,26 @@ export default class Thread extends React.Component { let pushshiftCommentIDs Promise.all([ - // Get thread from reddit - getThread(subreddit, threadID) - .then(extractHead) - .then(thread => { - this.setState({ thread }) - // Fetch the thread from pushshift if it was deleted/removed - if (isDeleted(thread.selftext)) { - getRemovedThread(threadID) - .then(removedThread => { - removedThread.removed = true - this.setState({ thread: removedThread }) - }) - } - }) - // Get comment ids from reddit - .then(() => getCommentIDs(subreddit, threadID)), + + // OUTDATED + // Get thread from reddit + // getThread(subreddit, threadID) + // .then(extractHead) + // .then(thread => { + // this.setState({ thread }) + + // // Fetch the thread from pushshift if it was deleted/removed + // if (isDeleted(thread.selftext)) { + // getRemovedThread(threadID) + // .then(removedThread => { + // removedThread.removed = true + // this.setState({ thread: removedThread }) + // }) + // } + // }) + // // Get comment ids from reddit + // .then(() => getCommentIDs(subreddit, threadID)), // Get comment ids from pushshift getAllCommentIDs(threadID) diff --git a/src/js/reddit/comment.js b/src/js/reddit/comment.js deleted file mode 100644 index f89b5ee..0000000 --- a/src/js/reddit/comment.js +++ /dev/null @@ -1,105 +0,0 @@ -import { json, fetchMultiple, jsonMultiple, unique } from 'utils' -import { redditAuth } from 'reddit/token' -import { getThread, extractPost } from './thread' - -export const commentLookup = {} - -const handleThreadComment = (comment, results) => { - if (comment.kind === 't1') { - // Has comment replies - if (comment.data.replies) { - // Handle all replies in the same way - comment.data.replies.data.children.forEach(reply => handleThreadComment(reply, results)) - delete comment.data.replies - } - - // Add to the return object - results.ids.push(comment.data.id) - // Store commment in lookup table - commentLookup[comment.data.id] = comment.data - } else if (comment.kind === 'more') { - if (comment.data.id === '_') { - // "continue this thread" comment - results.continueThisThreadIDs.push(comment.data.parent_id); - } else if (comment.data.children.length < comment.data.count) { - // "Load more"-comment (that is missing some of its children) - results.morechildrenIDs.push(comment.data.children) - } - - // Always add the "more"-comments children - results.ids.push(...comment.data.children) - } else { - console.error('WTF', comment.kind) - } - - return results -} - -const handleMoreChildren = (results, threadID) => ( - Promise.all(results.morechildrenIDs.map(idArray => - fetchMultiple(`https://oauth.reddit.com/api/morechildren?link_id=t3_${threadID}&children=`, idArray, redditAuth))) - .then(responsesArrays => Promise.all(responsesArrays.map(jsonMultiple))) - .then(responsesArrays => { - // Reset the array and parse for new morechildren-comments - results.morechildrenIDs = [] - responsesArrays.forEach(responses => responses.forEach(response => response.jquery[10][3][0].forEach(comment => { - handleThreadComment(comment, results) - }))) - }) - .then(() => { - if (results.morechildrenIDs.length !== 0) { - return handleMoreChildren(results, threadID) - } - - return Promise.resolve() - }) -) - -const extractComments = thread => thread[1].data.children - -const debug = res => { - console.log('IDs:', res.ids) - console.log('Continue:', res.continueThisThreadIDs) - console.log('Morechildren:', res.morechildrenIDs) -} - -const handleThread = thread => { - const comments = extractComments(thread) - - // Ugly "hack", using this object as a pointer for storing comments. - // Not very js-like - const results = { - ids: [], - continueThisThreadIDs: [], - morechildrenIDs: [], - } - - comments.map(comment => handleThreadComment(comment, results)) - const { subreddit, id: threadID } = extractPost(thread) - - return Promise.all([ - ...results.continueThisThreadIDs.map(id => getThread(subreddit, threadID, id) - .then(subthread => handleThread(subthread))), - handleMoreChildren(results, threadID), - ]) - .then(resultsArray => { - // Removed handleMoreChildren element - // This will result in resultsArray containing a list of comments - // from all 'continue this thread' posts - console.log(resultsArray) - resultsArray.pop() - console.log(results) - console.log(resultsArray) - }) - // zip - // ) - // .then( - // return results - // ) -} - - -export const getCommentIDs = (subreddit, threadID) => ( - getThread(subreddit, threadID) - .then(thread => handleThread(thread)) -) diff --git a/webpack.common.js b/webpack.common.js index 6025d7f..e6ab2be 100644 --- a/webpack.common.js +++ b/webpack.common.js @@ -1,14 +1,13 @@ const path = require('path'); +const ExtractTextPlugin = require('extract-text-webpack-plugin'); const HtmlWebpackPlugin = require('html-webpack-plugin'); const CopyWebpackPlugin = require('copy-webpack-plugin'); const CleanWebpackPlugin = require('clean-webpack-plugin'); -const ExtractTextPlugin = require('extract-text-webpack-plugin'); const extractSass = new ExtractTextPlugin({ filename: 'bundle.[contenthash].css', }); - module.exports = { entry: [ '@babel/polyfill',