prehistoric-systems/removeddit
1
0
Fork 0
mirror of https://github.com/gurnec/removeddit.git synced 2026-03-11 08:54:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fetch comment ids from pushshift parallel to reddit thread

Browse source
This commit is contained in:
Jesper Wrang 2018-01-14 22:25:54 +01:00
parent 62ab281c58
commit 9c3514253c
7 changed files with 95 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/js/components/CommentSection.js
View file

@ -1,7 +1,11 @@
import React from 'react'
export default (props) => (
<div id={props.root}>
{props.children}
</div>
)
export default (props) => {
const { children } = props
return (
<div>
<h1>{props.root}</h1>
<h1>{children}</h1>
</div>
)
}

20
src/js/components/StatusBox.js
View file

@ -1,18 +1,14 @@
import React from 'react'
import { connect } from 'react-redux'
class StatusBox extends React.Component {
render() {
return (
<div id='status'>
{this.props.text &&
<p id='status-text'>{this.props.text}</p>}
{this.props.image &&
<img id='status-image' src={this.props.image} />}
</div>
)
}
}
const StatusBox = props => (
<div id='status'>
{props.text &&
<p id='status-text'>{props.text}</p>}
{props.image &&
<img id='status-image' src={props.image} alt='status' />}
</div>
)
const mapStateToProps = state => ({
text: state.status.text,

4
src/js/components/ThreadHead.js
View file

@ -1,5 +1,5 @@
import React from 'react'
import { prettyScore, prettyDate, parse, redditThumbnails } from 'utils'
import { prettyScore, prettyDate, parse, redditThumbnails, isDeleted } from 'utils'
export default (props) => {
if (!props.title) {
@ -7,7 +7,7 @@ export default (props) => {
}
const url = props.url.replace('https://www.reddit.com', '')
const userLink = props.author !== '[deleted]' ? `https://www.reddit.com/user/${props.author}` : undefined
const userLink = isDeleted(props.author) ? undefined : `https://www.reddit.com/user/${props.author}`
let thumbnail
const thumbnailWidth = props.thumbnail_width ? props.thumbnail_width * 0.5 : 70

5
src/js/index.js
View file

@ -3,6 +3,7 @@ import ReactDOM from 'react-dom'
import { Provider } from 'react-redux'
import { store } from 'state'
import App from 'App'
// import { setStatusLoading } from 'state'
import '../sass/main.sass'
@ -13,8 +14,4 @@ ReactDOM.render(
document.getElementById('app')
)
// import {setStatusLoading} from 'state'
// store.dispatch(setStatusLoading('hi'))
// import { getThread } from 'reddit'
// getThread('videos', '7q4vxi')

50
src/js/pages/Thread.js
View file

@ -1,7 +1,14 @@
import React from 'react'
import ThreadHead from 'components/ThreadHead'
import { getThread } from 'reddit'
import { getThread as getRemovedThread } from 'pushshift'
import CommentSection from 'components/CommentSection'
import {
getThread,
getCommentIDs,
} from 'reddit'
import {
getThread as getRemovedThread,
getCommentIDs as getAllCommentIDs,
} from 'pushshift'
import { isDeleted } from 'utils'
export default class Thread extends React.Component {
@ -10,24 +17,39 @@ export default class Thread extends React.Component {
this.state = {
thread: {},
comments: [],
}
}
componentDidMount() {
const { subreddit, threadID } = this.props.match.params
getThread(subreddit, threadID)
.then(thread => {
this.setState({ thread })
let pushshiftCommentIDs
// Fetch the thread from pushshift if it was deleted/removed
if (isDeleted(thread.selftext)) {
getRemovedThread(threadID)
.then(removedThread => {
removedThread.removed = true
this.setState({ thread: removedThread })
})
}
Promise.all([
// Get thread from reddit
getThread(subreddit, threadID)
.then(thread => {
this.setState({ thread })
// Fetch the thread from pushshift if it was deleted/removed
if (isDeleted(thread.selftext)) {
getRemovedThread(threadID)
.then(removedThread => {
removedThread.removed = true
this.setState({ thread: removedThread })
})
}
}),
// Get comment ids from pushshift
getAllCommentIDs(threadID)
.then(commentIDs => { pushshiftCommentIDs = commentIDs }),
])
.then(() => {
// Get comment ids from reddit
getCommentIDs()
// .then(console.log)
})
}
@ -35,7 +57,7 @@ export default class Thread extends React.Component {
return (
<div>
<ThreadHead {...this.state.thread} />
<h1>lol</h1>
<CommentSection root={this.state.thread.id} comments={this.state.comments} />
</div>
)
}

4
src/js/pushshift/index.js
View file

@ -25,6 +25,10 @@ export const getThread = threadID => {
fetch(threadURL + JSON.stringify(elasticQuery))
.then(json)
.then(jsonData => jsonData.hits.hits[0]._source)
.then(thread => {
thread.id = toBase36(thread.id)
return thread
})
)
}

73
src/js/reddit/index.js
View file

@ -16,11 +16,11 @@ let baseCommentTree = {}
// Headers for getting reddit api token
const tokenInit = {
headers: {
Authorization: `Basic ${btoa(clientID + ':')}`,
Authorization: `Basic ${btoa(`${clientID}:`)}`,
'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
},
method: 'POST',
body: `grant_type=${encodeURIComponent('https://oauth.reddit.com/grants/installed_client')}&device_id=DO_NOT_TRACK_THIS_DEVICE`
body: `grant_type=${encodeURIComponent('https://oauth.reddit.com/grants/installed_client')}&device_id=DO_NOT_TRACK_THIS_DEVICE`,
}
const fetchToken = () => {
@ -59,17 +59,17 @@ export const getCommentIDs = () => {
// return HandleIDs.morechildren()
// .catch(function(error){
// return Promise.reject("Could not get comments from Reddit (moreChildren)");
// });
// });
// })
// .then(function(){
// return Promise.all(_.map(_.uniq(Comments.countinuethread), function(id) {
// return Promise.all(_.map(_.uniq(Comments.countinuethread), function(id) {
// return fetch(URLs.thread+"/_/"+id.split("_")[1], Reddit.init)
// .then(Fetch2.json)
// .catch(function(error){ return Promise.reject("Could not get comments from Reddit (continueThisThread)") })
// }))
// .catch(function(error){
// return Promise.reject("Could not get comments from Reddit (continueThisThread)");
// });
// });
// })
// .then(function(smallerThreads){
// _.forEach(smallerThreads, function(thread){
@ -81,7 +81,7 @@ export const getCommentIDs = () => {
// return Fetch2.multiple(URLs.format(URLs.pushshiftComments, Comments.removed), null, "data")
// .catch(function(error){
// return Promise.reject("Could not get removed comments");
// });
// });
// })
// .then(function(removedComments){
// Status.loading("Generating comments...");
@ -108,13 +108,13 @@ export const getCommentIDs = () => {
// var lookup = {};
// var toBeCreated = [];
// var getParentComments = function(toLookup){
// var getParentComments = function(toLookup){
// var newCommentsToLookup = [];
// var commentsToFetch = [];
// _.forEach(toLookup, function(id){
// var parentID = lookup[id].parent_id.split("_")[1];
// if(parentID === Reddit.threadID) {} // Has no parent (is parent of thread)
// else if(_.includes(Comments.toBeCreated, parentID)) {} // Parent already exists, do nothing
// else if(_.includes(newCommentsToLookup, parentID)) {} // Parent already exists (this iteration)
@ -123,11 +123,11 @@ export const getCommentIDs = () => {
// } else{
// commentsToFetch.push(parentID);
// }
// Comments.toBeCreated.push(id);
// });
// return new Promise(function(resolve, reject){
// return new Promise(function(resolve, reject){
// if(_.uniq(commentsToFetch).length !== 0) {
// fetch(URLs.singleComments + _.join(_.map(_.uniq(commentsToFetch),function(comments){
// return "t1_" + comments;
@ -142,9 +142,9 @@ export const getCommentIDs = () => {
// .then(function(){
// resolve();
// });
// } else {
// resolve();
// resolve();
// }
// })
// .then(function(){
@ -152,14 +152,14 @@ export const getCommentIDs = () => {
// return getParentComments(newCommentsToLookup)
// }
// });
// };
// return {
// ids: [], // The comments we found
// morechildren: [],
// countinuethread: [],
// allIDs: [], // All the comments that we were suppose to find
// removed: [],
// deleted: [],
@ -180,7 +180,7 @@ export const getCommentIDs = () => {
// if(_.has(Comments.lookup, Reddit.permalink)) {
// return Comments.lookup[Reddit.permalink].parent_id.split("_")[1];
// }
// }
// return "";
// },
// generate: function(removedComments) {
@ -188,26 +188,25 @@ export const getCommentIDs = () => {
// if(_.includes(Comments.deleted, comment.id)) {
// comment["deleted"] = true;
// } else {
// comment["removed"] = true;
// comment["removed"] = true;
// }
// Comments.lookup[comment.id] = comment;
// });
// Comments.removed = _.map(removedComments, function(comment){
// return comment.id;
// });
// return getParentComments(Comments.removed)
// .then(function(){
// ThreadHTML.createCommentSection();
// .then(function(){
// ThreadHTML.createCommentSection();
// ThreadHTML.createComments();
// })
// }
// }})();
// // ------------------------------------------------------------------------------
// // ----------------- Handle comments from different requests --------------------
// // ------------------------------------------------------------------------------
@ -217,7 +216,7 @@ export const getCommentIDs = () => {
// };
// var morechildren = function(){
// return Promise.all(_.map(_.uniq(Comments.morechildren), function(idArray){
// return Promise.all(_.map(_.uniq(Comments.morechildren), function(idArray){
// return Fetch2.multiple(URLs.format(URLs.moreChildren, idArray), Reddit.init);
// }))
// .then(function(responseArrays){
@ -232,7 +231,7 @@ export const getCommentIDs = () => {
// }).then(function(){
// if(Comments.morechildren.length !== 0) {
// return morechildren();
// }
// }
// });
// };
@ -243,7 +242,7 @@ export const getCommentIDs = () => {
// if(! _.has(Comments.lookup, id)) {
// return;
// }
// if(Comments.lookup[id].body === "[removed]") {
// Comments.removed.push(id);
// } else if (Comments.lookup[id].body === "[deleted]"){
@ -269,8 +268,8 @@ export const getCommentIDs = () => {
// var Extract = (function(){
// var normal = function(comment){
// var data = comment.data;
// if(comment.kind == "more") { // "Show more"-comment
// if(comment.kind == "more") { // "Show more"-comment
// if(data.id === "_") { // = "continue this thread" comment
// Comments.countinuethread.push(data.parent_id);
// } else if(data.children.length < data.count){ // "Load more"-comment (that is missing some of its children)
@ -284,15 +283,15 @@ export const getCommentIDs = () => {
// });
// delete data.replies;
// }
// Comments.ids.push(data.id);
// Comments.lookup[data.id] = data;
// }
// };
// return {
// normal: normal
// };
// };
// })();
@ -300,7 +299,7 @@ export const getCommentIDs = () => {
// // ---------------------------- Generating HTML ---------------------------------
// // ------------------------------------------------------------------------------
// var createComments = function(){
// var commentsToCreate = _.sortBy(_.uniq(Comments.toBeCreated), function(id) {
// return Comments.lookup[id].score;
@ -308,14 +307,14 @@ export const getCommentIDs = () => {
// var createdComments = [Comments.getRoot()];
// var didSomething = false;
// while(commentsToCreate.length > 0) {
// didSomething = false;
// for(var i = commentsToCreate.length - 1; i >= 0; i--) {
// var id = commentsToCreate[i];
// var parentID = Comments.lookup[id].parent_id.split("_")[1];
// if(_.includes(createdComments, parentID)) {
// document.getElementById(parentID).appendChild(createComment(Comments.lookup[id]));
// createdComments.push(id);
@ -323,7 +322,7 @@ export const getCommentIDs = () => {
// didSomething = true;
// }
// }
// // Fail safe (parents missing for the rest of the comments, shouldn't happend but oh well :D)
// if(!didSomething) {
// console.error("Didn't generate all comments correctly");
@ -332,5 +331,3 @@ export const getCommentIDs = () => {
// }
// };