diff --git a/.eslintrc.json b/.eslintrc.json index 6909e48..5e36fe5 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -18,6 +18,7 @@ "import/extensions": "off", "import/no-extraneous-dependencies": "off", "import/no-unresolved": "off", + "import/prefer-default-export": "off", "jsx-a11y/anchor-has-content": "off", "jsx-a11y/anchor-is-valid": [ "error", @@ -46,6 +47,7 @@ "allowForLoopAfterthoughts": true } ], + "no-prototype-builtins": "off", "no-script-url": "off", "no-underscore-dangle": "off", "no-unused-vars": "warn", diff --git a/src/index.html b/src/index.html index 1226026..97bb53c 100644 --- a/src/index.html +++ b/src/index.html @@ -10,6 +10,7 @@ +
\ No newline at end of file diff --git a/src/js/pages/Thread.js b/src/js/pages/Thread.js index f389911..5a32347 100644 --- a/src/js/pages/Thread.js +++ b/src/js/pages/Thread.js @@ -4,6 +4,7 @@ import CommentSection from 'components/CommentSection' import { getThread, getCommentIDs, + extractHead, } from 'reddit' import { getThread as getRemovedThread, @@ -23,12 +24,13 @@ export default class Thread extends React.Component { componentDidMount() { const { subreddit, threadID } = this.props.match.params - + console.timeEnd('scripts loaded') let pushshiftCommentIDs Promise.all([ // Get thread from reddit getThread(subreddit, threadID) + .then(extractHead) .then(thread => { this.setState({ thread }) @@ -40,15 +42,15 @@ export default class Thread extends React.Component { this.setState({ thread: removedThread }) }) } - }), + }) + // Get comment ids from reddit + .then(() => getCommentIDs(subreddit, threadID)), // Get comment ids from pushshift getAllCommentIDs(threadID) .then(commentIDs => { pushshiftCommentIDs = commentIDs }), ]) .then(() => { - // Get comment ids from reddit - getCommentIDs() // .then(console.log) }) } diff --git a/src/js/reddit/comment.js b/src/js/reddit/comment.js new file mode 100644 index 0000000..6381c83 --- /dev/null +++ b/src/js/reddit/comment.js @@ -0,0 +1,104 @@ +import { json, fetchMultiple, jsonMultiple, unique } from 'utils' +import { redditAuth } from 'reddit/token' +import { getThread, extractPost } from './thread' + +export const commentLookup = {} + +const handleThreadComment = (comment, results) => { + if (comment.kind === 't1') { + // Has comment replies + if (comment.data.replies) { + // Handle all replies in the same way + comment.data.replies.data.children.forEach(reply => handleThreadComment(reply, results)) + delete comment.data.replies + } + + // Add to the return object + results.ids.push(comment.data.id) + // Store commment in lookup table + commentLookup[comment.data.id] = comment.data + } else if (comment.kind === 'more') { + if (comment.data.id === '_') { + // "continue this thread" comment + results.continueThisThreadIDs.push(comment.data.parent_id); + } else if (comment.data.children.length < comment.data.count) { + // "Load more"-comment (that is missing some of its children) + results.morechildrenIDs.push(comment.data.children) + } + + // Always add the "more"-comments children + results.ids.push(...comment.data.children) + } else { + console.error('WTF', comment.kind) + } + + return results +} + +const handleMoreChildren = (results, threadID) => ( + // console.log(results.morechildrenIDs.map(idArray => idArray)) + Promise.all(results.morechildrenIDs.map(idArray => + fetchMultiple(`https://oauth.reddit.com/api/morechildren?link_id=t3_${threadID}&children=`, idArray, redditAuth))) + .then(responsesArrays => Promise.all(responsesArrays.map(jsonMultiple))) + .then(responsesArrays => { + // Reset the array and parse for new morechildren-comments + results.morechildrenIDs = [] + responsesArrays.forEach(responses => responses.forEach(response => response.jquery[10][3][0].forEach(comment => { + handleThreadComment(comment, results) + }))) + }) + .then(() => { + if (results.morechildrenIDs.length !== 0) { + return handleMoreChildren(results, threadID) + } + + return Promise.resolve() + }) +) + +const extractComments = thread => thread[1].data.children + +const debug = res => { + console.log('IDs:', res.ids) + console.log('Continue:', res.continueThisThreadIDs) + console.log('Morechildren:', res.morechildrenIDs) +} + +const handleThread = thread => { + const comments = extractComments(thread) + + // Ugly "hack", using this object as a pointer for storing comments. + // Not very js-like + const results = { + ids: [], + continueThisThreadIDs: [], + morechildrenIDs: [], + } + + comments.map(comment => handleThreadComment(comment, results)) + const { subreddit, id: threadID } = extractPost(thread) + + return Promise.all([ + ...results.continueThisThreadIDs.map(id => getThread(subreddit, threadID, id)), + // .then(subthreads => subthreads.map(subthread => handleThread(subthread)))), + handleMoreChildren(results, threadID), + ]) + .then(resultsArray => { + // Removed handleMoreChildren element + // This will result in resultsArray containing a list of comments + // from all 'continue this thread' posts + console.log(resultsArray) + resultsArray.pop() + }) + // zip + // ) + // .then( + // return results + // ) +} + + +export const getCommentIDs = (subreddit, threadID) => ( + getThread(subreddit, threadID) + .then(thread => handleThread(thread)) +) diff --git a/src/js/reddit/index.js b/src/js/reddit/index.js index d0a78c5..e195096 100644 --- a/src/js/reddit/index.js +++ b/src/js/reddit/index.js @@ -1,79 +1,5 @@ -import { json, flatten } from 'utils' -import { fetchToken, redditAuth } from 'reddit/token' - -let cachedThread = {} -export const lookup = {} - -export const getThread = (subreddit, threadID) => ( - fetchToken() - .then(() => fetch(`https://oauth.reddit.com/r/${subreddit}/comments/${threadID}`, redditAuth)) - .then(json) - .then(results => { - // Save the comments for later - cachedThread = results - - // Return the thread - return results[0].data.children[0].data - }) -) - -const handleThreadComment = (comment, results) => { - if (comment.kind === 't1') { - // Has comment replies - if (comment.data.replies) { - // Handle all replies in the same way - comment.data.replies.data.children.forEach(reply => handleThreadComment(reply, results)) - delete comment.data.replies - } - - // Add to the return object - results.ids.push(comment.data.id) - // Store commment in lookup table - lookup[comment.data.id] = comment.data - } else if (comment.kind === 'more') { - if (comment.data.id === '_') { - // "continue this thread" comment - results.continueThisThreadIDs.push(comment.data.parent_id); - } else if (comment.data.children.length < comment.data.count) { - // "Load more"-comment (that is missing some of its children) - results.morechildrenIDs.push(comment.data.children) - } - - // Always add the "more"-comments children - results.ids.push(...comment.data.children) - } else { - console.error('WTF', comment.kind) - } - - return results -} - -const handleMoreChildren = () => { - -} - -const handleThread = thread => { - const comments = thread[1].data.children - - // Ugly "hack", using this object as a pointer for storing comments. - // Not very js-like - const results = { - ids: [], - continueThisThreadIDs: [], - morechildrenIDs: [], - } - - comments.map(comment => handleThreadComment(comment, results)) - const { ids, continueThisThreadIDs, morechildrenIDs } = results - - console.log(ids) - console.log(continueThisThreadIDs) - console.log(morechildrenIDs) - - handleMoreChildren() -} - -export const getCommentIDs = () => handleThread(cachedThread) +export { getThread, extractPost } from './thread' +export { getCommentIDs } from './comment' // return HandleIDs.morechildren() // .catch(function(error){ @@ -229,11 +155,6 @@ export const getCommentIDs = () => handleThread(cachedThread) // // ------------------------------------------------------------------------------ // // ----------------- Handle comments from different requests -------------------- // // ------------------------------------------------------------------------------ -// var HandleIDs = (function(){ -// var normal = function(thread){ -// return _.flatten(_.map(thread[1].data.children, Extract.normal)) -// }; - // var morechildren = function(){ // return Promise.all(_.map(_.uniq(Comments.morechildren), function(idArray){ // return Fetch2.multiple(URLs.format(URLs.moreChildren, idArray), Reddit.init); diff --git a/src/js/reddit/thread.js b/src/js/reddit/thread.js new file mode 100644 index 0000000..0958b8f --- /dev/null +++ b/src/js/reddit/thread.js @@ -0,0 +1,36 @@ +import { json } from 'utils' +import { fetchToken, redditAuth } from 'reddit/token' + +const cachedThreads = {} + +export const getThread = (subreddit, threadID, continueThisThreadID = '') => { + // We have already downloaded the thread and can use a cached copy + if (cachedThreads.hasOwnProperty(threadID)) { + if (cachedThreads[threadID].hasOwnProperty(continueThisThreadID)) { + return Promise.resolve(cachedThreads[threadID][continueThisThreadID]) + } + } + + const url = `https://oauth.reddit.com/r/${subreddit}/comments/${threadID}/_/${continueThisThreadID}` + // Fetch thread from reddit + return ( + fetchToken() + .then(() => fetch(url, redditAuth)) + .then(json) + .then(thread => { + // Save the thread for later + if (!cachedThreads.hasOwnProperty(threadID)) { + cachedThreads[threadID] = {} + } + + cachedThreads[threadID][continueThisThreadID] = thread + + // Return the thread + return thread + }) + ) +} + +// Thread = Post + Comments +// Return the post itself +export const extractPost = thread => thread[0].data.children[0].data diff --git a/src/js/utils/index.js b/src/js/utils/index.js index e199291..2b3d90f 100644 --- a/src/js/utils/index.js +++ b/src/js/utils/index.js @@ -8,6 +8,30 @@ export const flatten = arr => arr.reduce( [] ) +export const unique = arr => arr.filter((value, index) => arr.indexOf(value) === index) + +// Take on big array and split it into an array of chunks with correct size +export const chunk = (arr, size) => { + const chunks = [] + for (let i = 0; i < arr.length; i += size) { + chunks.push(arr.slice(i, i + size)) + } + return chunks +} + +// JSON parsing for fetch +export const json = x => x.json() + +// Make multiple requests to the same url, with an array of data (usually comment IDs) +// This is needed since there is a limit on how long a url can be +export const fetchMultiple = (url, arr, header, size = 100) => { + const subArrays = chunk(arr, size) + + return Promise.all(subArrays.map(subArr => fetch(url + subArr.join(), header))) +} + +export const jsonMultiple = responses => Promise.all(responses.map(json)) + // Change bases export const toBase36 = number => parseInt(number, 10).toString(36) export const toBase10 = numberString => parseInt(numberString, 36) @@ -18,9 +42,6 @@ export const isDeleted = testString => testString === '[deleted]' // Default thumbnails for reddit threads export const redditThumbnails = ['self', 'default', 'image', 'nsfw'] -// JSON parsing for fetch -export const json = x => x.json() - // Parse comments export const parse = text => markdown.render(text) diff --git a/webpack.common.js b/webpack.common.js index c83f1d5..6025d7f 100644 --- a/webpack.common.js +++ b/webpack.common.js @@ -5,71 +5,71 @@ const CleanWebpackPlugin = require('clean-webpack-plugin'); const ExtractTextPlugin = require('extract-text-webpack-plugin'); const extractSass = new ExtractTextPlugin({ - filename: 'bundle.[contenthash].css', + filename: 'bundle.[contenthash].css', }); module.exports = { - entry: [ - '@babel/polyfill', - 'whatwg-fetch', - './src/js/index.js', - ], - output: { - path: path.resolve(__dirname, 'dist'), - filename: 'bundle.[chunkhash].js', - publicPath: '/' - }, - resolve: { - modules: [ - path.resolve('./src/js'), - path.resolve('./node_modules') - ] - }, - module: { - loaders: [ - { - test: /\.js$/, - exclude: /node_modules/, - use: [ - { - loader: 'babel-loader', - options: { - cacheDirectory: true - } - } - ] - }, - { - test: /\.html$/, - loader: 'html-loader' - }, - { - test: /\.sass$/, - loader: extractSass.extract({ - use: ['css-loader','sass-loader'] - }) - } - ] - }, - plugins: [ - new CleanWebpackPlugin(['dist']), - extractSass, - new HtmlWebpackPlugin({ - template: 'src/index.html' - }), - new CopyWebpackPlugin([ - { - from: 'src/404.html', - to: '404.html' - }, - { - from: 'src/images', - to: 'images/' - } - ]), - ], - stats: { - colors: true - }, -} \ No newline at end of file + entry: [ + '@babel/polyfill', + 'whatwg-fetch', + './src/js/index.js', + ], + output: { + path: path.resolve(__dirname, 'dist'), + filename: 'bundle.[chunkhash].js', + publicPath: '/', + }, + resolve: { + modules: [ + path.resolve('./src/js'), + path.resolve('./node_modules'), + ], + }, + module: { + loaders: [ + { + test: /\.js$/, + exclude: /node_modules/, + use: [ + { + loader: 'babel-loader', + options: { + cacheDirectory: true, + }, + }, + ], + }, + { + test: /\.html$/, + loader: 'html-loader', + }, + { + test: /\.sass$/, + loader: extractSass.extract({ + use: ['css-loader', 'sass-loader'], + }), + }, + ], + }, + plugins: [ + new CleanWebpackPlugin(['dist']), + extractSass, + new HtmlWebpackPlugin({ + template: 'src/index.html', + }), + new CopyWebpackPlugin([ + { + from: 'src/404.html', + to: '404.html', + }, + { + from: 'src/images', + to: 'images/', + }, + ]), + ], + stats: { + colors: true, + }, +} diff --git a/webpack.dev.js b/webpack.dev.js index 593ca2c..4df4515 100644 --- a/webpack.dev.js +++ b/webpack.dev.js @@ -3,7 +3,7 @@ const merge = require('webpack-merge'); const common = require('./webpack.common.js'); module.exports = merge(common, { - devServer: { - historyApiFallback: true - }, -}); \ No newline at end of file + devServer: { + historyApiFallback: true, + }, +}); diff --git a/webpack.prod.js b/webpack.prod.js index 66f7c26..81fc0c9 100644 --- a/webpack.prod.js +++ b/webpack.prod.js @@ -4,10 +4,10 @@ const common = require('./webpack.common.js'); const UglifyJSPlugin = require('uglifyjs-webpack-plugin'); module.exports = merge(common, { - plugins: [ - new UglifyJSPlugin(), - new webpack.DefinePlugin({ - 'process.env.NODE_ENV': JSON.stringify('production') - }) - ] -}); \ No newline at end of file + plugins: [ + new UglifyJSPlugin(), + new webpack.DefinePlugin({ + 'process.env.NODE_ENV': JSON.stringify('production'), + }), + ], +});