mirror of
https://github.com/gurnec/removeddit.git
synced 2026-03-11 08:54:27 +00:00
Can now parse 'morechildren'-comments
This commit is contained in:
Jesper Wrang
parent
4492064184
commit
9bab14ffcb
10 changed files with 250 additions and 163 deletions
|
|
@ -18,6 +18,7 @@
|
|||
"import/extensions": "off",
|
||||
"import/no-extraneous-dependencies": "off",
|
||||
"import/no-unresolved": "off",
|
||||
"import/prefer-default-export": "off",
|
||||
"jsx-a11y/anchor-has-content": "off",
|
||||
"jsx-a11y/anchor-is-valid": [
|
||||
"error",
|
||||
|
|
@ -46,6 +47,7 @@
|
|||
"allowForLoopAfterthoughts": true
|
||||
}
|
||||
],
|
||||
"no-prototype-builtins": "off",
|
||||
"no-script-url": "off",
|
||||
"no-underscore-dangle": "off",
|
||||
"no-unused-vars": "warn",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
<link href="/images/favicon.ico" rel="shotcut icon">
|
||||
</head>
|
||||
<body>
|
||||
<script>console.time('scripts loaded')</script>
|
||||
<div id="app"></div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -4,6 +4,7 @@ import CommentSection from 'components/CommentSection'
|
|||
import {
|
||||
getThread,
|
||||
getCommentIDs,
|
||||
extractHead,
|
||||
} from 'reddit'
|
||||
import {
|
||||
getThread as getRemovedThread,
|
||||
|
|
@ -23,12 +24,13 @@ export default class Thread extends React.Component {
|
|||
|
||||
componentDidMount() {
|
||||
const { subreddit, threadID } = this.props.match.params
|
||||
|
||||
console.timeEnd('scripts loaded')
|
||||
let pushshiftCommentIDs
|
||||
|
||||
Promise.all([
|
||||
// Get thread from reddit
|
||||
getThread(subreddit, threadID)
|
||||
.then(extractHead)
|
||||
.then(thread => {
|
||||
this.setState({ thread })
|
||||
|
||||
|
|
@ -40,15 +42,15 @@ export default class Thread extends React.Component {
|
|||
this.setState({ thread: removedThread })
|
||||
})
|
||||
}
|
||||
}),
|
||||
})
|
||||
// Get comment ids from reddit
|
||||
.then(() => getCommentIDs(subreddit, threadID)),
|
||||
|
||||
// Get comment ids from pushshift
|
||||
getAllCommentIDs(threadID)
|
||||
.then(commentIDs => { pushshiftCommentIDs = commentIDs }),
|
||||
])
|
||||
.then(() => {
|
||||
// Get comment ids from reddit
|
||||
getCommentIDs()
|
||||
// .then(console.log)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
104
src/js/reddit/comment.js
Normal file
104
src/js/reddit/comment.js
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import { json, fetchMultiple, jsonMultiple, unique } from 'utils'
|
||||
import { redditAuth } from 'reddit/token'
|
||||
import { getThread, extractPost } from './thread'
|
||||
|
||||
export const commentLookup = {}
|
||||
|
||||
const handleThreadComment = (comment, results) => {
|
||||
if (comment.kind === 't1') {
|
||||
// Has comment replies
|
||||
if (comment.data.replies) {
|
||||
// Handle all replies in the same way
|
||||
comment.data.replies.data.children.forEach(reply => handleThreadComment(reply, results))
|
||||
delete comment.data.replies
|
||||
}
|
||||
|
||||
// Add to the return object
|
||||
results.ids.push(comment.data.id)
|
||||
// Store commment in lookup table
|
||||
commentLookup[comment.data.id] = comment.data
|
||||
} else if (comment.kind === 'more') {
|
||||
if (comment.data.id === '_') {
|
||||
// "continue this thread" comment
|
||||
results.continueThisThreadIDs.push(comment.data.parent_id);
|
||||
} else if (comment.data.children.length < comment.data.count) {
|
||||
// "Load more"-comment (that is missing some of its children)
|
||||
results.morechildrenIDs.push(comment.data.children)
|
||||
}
|
||||
|
||||
// Always add the "more"-comments children
|
||||
results.ids.push(...comment.data.children)
|
||||
} else {
|
||||
console.error('WTF', comment.kind)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
const handleMoreChildren = (results, threadID) => (
|
||||
// console.log(results.morechildrenIDs.map(idArray => idArray))
|
||||
Promise.all(results.morechildrenIDs.map(idArray =>
|
||||
fetchMultiple(`https://oauth.reddit.com/api/morechildren?link_id=t3_${threadID}&children=`, idArray, redditAuth)))
|
||||
.then(responsesArrays => Promise.all(responsesArrays.map(jsonMultiple)))
|
||||
.then(responsesArrays => {
|
||||
// Reset the array and parse for new morechildren-comments
|
||||
results.morechildrenIDs = []
|
||||
responsesArrays.forEach(responses => responses.forEach(response => response.jquery[10][3][0].forEach(comment => {
|
||||
handleThreadComment(comment, results)
|
||||
})))
|
||||
})
|
||||
.then(() => {
|
||||
if (results.morechildrenIDs.length !== 0) {
|
||||
return handleMoreChildren(results, threadID)
|
||||
}
|
||||
|
||||
return Promise.resolve()
|
||||
})
|
||||
)
|
||||
|
||||
const extractComments = thread => thread[1].data.children
|
||||
|
||||
const debug = res => {
|
||||
console.log('IDs:', res.ids)
|
||||
console.log('Continue:', res.continueThisThreadIDs)
|
||||
console.log('Morechildren:', res.morechildrenIDs)
|
||||
}
|
||||
|
||||
const handleThread = thread => {
|
||||
const comments = extractComments(thread)
|
||||
|
||||
// Ugly "hack", using this object as a pointer for storing comments.
|
||||
// Not very js-like
|
||||
const results = {
|
||||
ids: [],
|
||||
continueThisThreadIDs: [],
|
||||
morechildrenIDs: [],
|
||||
}
|
||||
|
||||
comments.map(comment => handleThreadComment(comment, results))
|
||||
const { subreddit, id: threadID } = extractPost(thread)
|
||||
|
||||
return Promise.all([
|
||||
...results.continueThisThreadIDs.map(id => getThread(subreddit, threadID, id)),
|
||||
// .then(subthreads => subthreads.map(subthread => handleThread(subthread)))),
|
||||
handleMoreChildren(results, threadID),
|
||||
])
|
||||
.then(resultsArray => {
|
||||
// Removed handleMoreChildren element
|
||||
// This will result in resultsArray containing a list of comments
|
||||
// from all 'continue this thread' posts
|
||||
console.log(resultsArray)
|
||||
resultsArray.pop()
|
||||
})
|
||||
// zip
|
||||
// )
|
||||
// .then(
|
||||
// return results
|
||||
// )
|
||||
}
|
||||
|
||||
|
||||
export const getCommentIDs = (subreddit, threadID) => (
|
||||
getThread(subreddit, threadID)
|
||||
.then(thread => handleThread(thread))
|
||||
)
|
||||
|
|
@ -1,79 +1,5 @@
|
|||
import { json, flatten } from 'utils'
|
||||
import { fetchToken, redditAuth } from 'reddit/token'
|
||||
|
||||
let cachedThread = {}
|
||||
export const lookup = {}
|
||||
|
||||
export const getThread = (subreddit, threadID) => (
|
||||
fetchToken()
|
||||
.then(() => fetch(`https://oauth.reddit.com/r/${subreddit}/comments/${threadID}`, redditAuth))
|
||||
.then(json)
|
||||
.then(results => {
|
||||
// Save the comments for later
|
||||
cachedThread = results
|
||||
|
||||
// Return the thread
|
||||
return results[0].data.children[0].data
|
||||
})
|
||||
)
|
||||
|
||||
const handleThreadComment = (comment, results) => {
|
||||
if (comment.kind === 't1') {
|
||||
// Has comment replies
|
||||
if (comment.data.replies) {
|
||||
// Handle all replies in the same way
|
||||
comment.data.replies.data.children.forEach(reply => handleThreadComment(reply, results))
|
||||
delete comment.data.replies
|
||||
}
|
||||
|
||||
// Add to the return object
|
||||
results.ids.push(comment.data.id)
|
||||
// Store commment in lookup table
|
||||
lookup[comment.data.id] = comment.data
|
||||
} else if (comment.kind === 'more') {
|
||||
if (comment.data.id === '_') {
|
||||
// "continue this thread" comment
|
||||
results.continueThisThreadIDs.push(comment.data.parent_id);
|
||||
} else if (comment.data.children.length < comment.data.count) {
|
||||
// "Load more"-comment (that is missing some of its children)
|
||||
results.morechildrenIDs.push(comment.data.children)
|
||||
}
|
||||
|
||||
// Always add the "more"-comments children
|
||||
results.ids.push(...comment.data.children)
|
||||
} else {
|
||||
console.error('WTF', comment.kind)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
const handleMoreChildren = () => {
|
||||
|
||||
}
|
||||
|
||||
const handleThread = thread => {
|
||||
const comments = thread[1].data.children
|
||||
|
||||
// Ugly "hack", using this object as a pointer for storing comments.
|
||||
// Not very js-like
|
||||
const results = {
|
||||
ids: [],
|
||||
continueThisThreadIDs: [],
|
||||
morechildrenIDs: [],
|
||||
}
|
||||
|
||||
comments.map(comment => handleThreadComment(comment, results))
|
||||
const { ids, continueThisThreadIDs, morechildrenIDs } = results
|
||||
|
||||
console.log(ids)
|
||||
console.log(continueThisThreadIDs)
|
||||
console.log(morechildrenIDs)
|
||||
|
||||
handleMoreChildren()
|
||||
}
|
||||
|
||||
export const getCommentIDs = () => handleThread(cachedThread)
|
||||
export { getThread, extractPost } from './thread'
|
||||
export { getCommentIDs } from './comment'
|
||||
|
||||
// return HandleIDs.morechildren()
|
||||
// .catch(function(error){
|
||||
|
|
@ -229,11 +155,6 @@ export const getCommentIDs = () => handleThread(cachedThread)
|
|||
// // ------------------------------------------------------------------------------
|
||||
// // ----------------- Handle comments from different requests --------------------
|
||||
// // ------------------------------------------------------------------------------
|
||||
// var HandleIDs = (function(){
|
||||
// var normal = function(thread){
|
||||
// return _.flatten(_.map(thread[1].data.children, Extract.normal))
|
||||
// };
|
||||
|
||||
// var morechildren = function(){
|
||||
// return Promise.all(_.map(_.uniq(Comments.morechildren), function(idArray){
|
||||
// return Fetch2.multiple(URLs.format(URLs.moreChildren, idArray), Reddit.init);
|
||||
|
|
|
|||
36
src/js/reddit/thread.js
Normal file
36
src/js/reddit/thread.js
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import { json } from 'utils'
|
||||
import { fetchToken, redditAuth } from 'reddit/token'
|
||||
|
||||
const cachedThreads = {}
|
||||
|
||||
export const getThread = (subreddit, threadID, continueThisThreadID = '') => {
|
||||
// We have already downloaded the thread and can use a cached copy
|
||||
if (cachedThreads.hasOwnProperty(threadID)) {
|
||||
if (cachedThreads[threadID].hasOwnProperty(continueThisThreadID)) {
|
||||
return Promise.resolve(cachedThreads[threadID][continueThisThreadID])
|
||||
}
|
||||
}
|
||||
|
||||
const url = `https://oauth.reddit.com/r/${subreddit}/comments/${threadID}/_/${continueThisThreadID}`
|
||||
// Fetch thread from reddit
|
||||
return (
|
||||
fetchToken()
|
||||
.then(() => fetch(url, redditAuth))
|
||||
.then(json)
|
||||
.then(thread => {
|
||||
// Save the thread for later
|
||||
if (!cachedThreads.hasOwnProperty(threadID)) {
|
||||
cachedThreads[threadID] = {}
|
||||
}
|
||||
|
||||
cachedThreads[threadID][continueThisThreadID] = thread
|
||||
|
||||
// Return the thread
|
||||
return thread
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Thread = Post + Comments
|
||||
// Return the post itself
|
||||
export const extractPost = thread => thread[0].data.children[0].data
|
||||
|
|
@ -8,6 +8,30 @@ export const flatten = arr => arr.reduce(
|
|||
[]
|
||||
)
|
||||
|
||||
export const unique = arr => arr.filter((value, index) => arr.indexOf(value) === index)
|
||||
|
||||
// Take on big array and split it into an array of chunks with correct size
|
||||
export const chunk = (arr, size) => {
|
||||
const chunks = []
|
||||
for (let i = 0; i < arr.length; i += size) {
|
||||
chunks.push(arr.slice(i, i + size))
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
// JSON parsing for fetch
|
||||
export const json = x => x.json()
|
||||
|
||||
// Make multiple requests to the same url, with an array of data (usually comment IDs)
|
||||
// This is needed since there is a limit on how long a url can be
|
||||
export const fetchMultiple = (url, arr, header, size = 100) => {
|
||||
const subArrays = chunk(arr, size)
|
||||
|
||||
return Promise.all(subArrays.map(subArr => fetch(url + subArr.join(), header)))
|
||||
}
|
||||
|
||||
export const jsonMultiple = responses => Promise.all(responses.map(json))
|
||||
|
||||
// Change bases
|
||||
export const toBase36 = number => parseInt(number, 10).toString(36)
|
||||
export const toBase10 = numberString => parseInt(numberString, 36)
|
||||
|
|
@ -18,9 +42,6 @@ export const isDeleted = testString => testString === '[deleted]'
|
|||
// Default thumbnails for reddit threads
|
||||
export const redditThumbnails = ['self', 'default', 'image', 'nsfw']
|
||||
|
||||
// JSON parsing for fetch
|
||||
export const json = x => x.json()
|
||||
|
||||
// Parse comments
|
||||
export const parse = text => markdown.render(text)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,71 +5,71 @@ const CleanWebpackPlugin = require('clean-webpack-plugin');
|
|||
const ExtractTextPlugin = require('extract-text-webpack-plugin');
|
||||
|
||||
const extractSass = new ExtractTextPlugin({
|
||||
filename: 'bundle.[contenthash].css',
|
||||
filename: 'bundle.[contenthash].css',
|
||||
});
|
||||
|
||||
|
||||
module.exports = {
|
||||
entry: [
|
||||
'@babel/polyfill',
|
||||
'whatwg-fetch',
|
||||
'./src/js/index.js',
|
||||
],
|
||||
output: {
|
||||
path: path.resolve(__dirname, 'dist'),
|
||||
filename: 'bundle.[chunkhash].js',
|
||||
publicPath: '/'
|
||||
},
|
||||
resolve: {
|
||||
modules: [
|
||||
path.resolve('./src/js'),
|
||||
path.resolve('./node_modules')
|
||||
]
|
||||
},
|
||||
module: {
|
||||
loaders: [
|
||||
{
|
||||
test: /\.js$/,
|
||||
exclude: /node_modules/,
|
||||
use: [
|
||||
{
|
||||
loader: 'babel-loader',
|
||||
options: {
|
||||
cacheDirectory: true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
test: /\.html$/,
|
||||
loader: 'html-loader'
|
||||
},
|
||||
{
|
||||
test: /\.sass$/,
|
||||
loader: extractSass.extract({
|
||||
use: ['css-loader','sass-loader']
|
||||
})
|
||||
}
|
||||
]
|
||||
},
|
||||
plugins: [
|
||||
new CleanWebpackPlugin(['dist']),
|
||||
extractSass,
|
||||
new HtmlWebpackPlugin({
|
||||
template: 'src/index.html'
|
||||
}),
|
||||
new CopyWebpackPlugin([
|
||||
{
|
||||
from: 'src/404.html',
|
||||
to: '404.html'
|
||||
},
|
||||
{
|
||||
from: 'src/images',
|
||||
to: 'images/'
|
||||
}
|
||||
]),
|
||||
],
|
||||
stats: {
|
||||
colors: true
|
||||
},
|
||||
}
|
||||
entry: [
|
||||
'@babel/polyfill',
|
||||
'whatwg-fetch',
|
||||
'./src/js/index.js',
|
||||
],
|
||||
output: {
|
||||
path: path.resolve(__dirname, 'dist'),
|
||||
filename: 'bundle.[chunkhash].js',
|
||||
publicPath: '/',
|
||||
},
|
||||
resolve: {
|
||||
modules: [
|
||||
path.resolve('./src/js'),
|
||||
path.resolve('./node_modules'),
|
||||
],
|
||||
},
|
||||
module: {
|
||||
loaders: [
|
||||
{
|
||||
test: /\.js$/,
|
||||
exclude: /node_modules/,
|
||||
use: [
|
||||
{
|
||||
loader: 'babel-loader',
|
||||
options: {
|
||||
cacheDirectory: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
test: /\.html$/,
|
||||
loader: 'html-loader',
|
||||
},
|
||||
{
|
||||
test: /\.sass$/,
|
||||
loader: extractSass.extract({
|
||||
use: ['css-loader', 'sass-loader'],
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
plugins: [
|
||||
new CleanWebpackPlugin(['dist']),
|
||||
extractSass,
|
||||
new HtmlWebpackPlugin({
|
||||
template: 'src/index.html',
|
||||
}),
|
||||
new CopyWebpackPlugin([
|
||||
{
|
||||
from: 'src/404.html',
|
||||
to: '404.html',
|
||||
},
|
||||
{
|
||||
from: 'src/images',
|
||||
to: 'images/',
|
||||
},
|
||||
]),
|
||||
],
|
||||
stats: {
|
||||
colors: true,
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ const merge = require('webpack-merge');
|
|||
const common = require('./webpack.common.js');
|
||||
|
||||
module.exports = merge(common, {
|
||||
devServer: {
|
||||
historyApiFallback: true
|
||||
},
|
||||
});
|
||||
devServer: {
|
||||
historyApiFallback: true,
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,10 +4,10 @@ const common = require('./webpack.common.js');
|
|||
const UglifyJSPlugin = require('uglifyjs-webpack-plugin');
|
||||
|
||||
module.exports = merge(common, {
|
||||
plugins: [
|
||||
new UglifyJSPlugin(),
|
||||
new webpack.DefinePlugin({
|
||||
'process.env.NODE_ENV': JSON.stringify('production')
|
||||
})
|
||||
]
|
||||
});
|
||||
plugins: [
|
||||
new UglifyJSPlugin(),
|
||||
new webpack.DefinePlugin({
|
||||
'process.env.NODE_ENV': JSON.stringify('production'),
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue