Changed to a simpler way of parsing comments

.eslintrc

@@ -62,6 +62,7 @@
             "never"
         ],
         "no-use-before-define": "off",
-        "import/first": "off"
+        "import/first": "off",
+        "prefer-destructuring": "off"
     }
 }

README.md

@@ -7,7 +7,6 @@ This is a done by comparing the comments found from Reddit API and comments bein
 # Development
 ```
 sudo git clone https://github.com/JubbeArt/removeddit.git && cd removeddit
-sudo apt install npm
 npm install
 npm start
 ```

optimization

@@ -0,0 +1 @@
+take utc from thread and limit pushshift comments

src/js/api/pushshift/index.js

@@ -3,13 +3,6 @@ import { json, toBase10, toBase36, chunk, flatten } from 'utils'
 const baseURL = 'https://elastic.pushshift.io'
 const postURL = `${baseURL}/rs/submissions/_search?source=`
 const commentURL = `${baseURL}/rc/comments/_search?source=`
-const commentIDsURL = 'https://api.pushshift.io/reddit/submission/comment_ids/'
-
-export const getCommentIDs = threadID => (
-  fetch(commentIDsURL + threadID)
-    .then(json)
-    .then(results => results.data)
-)
 
 export const getPost = threadID => {
   const elasticQuery = {
@@ -31,17 +24,14 @@ export const getPost = threadID => {
   )
 }
 
-export const getComments = commentIDs => (
-  Promise.all(chunk(commentIDs, 100).map(fetchComments)).then(flatten)
-)
-
-const fetchComments = commentIDs => {
+export const getComments = threadID => {
   const elasticQuery = {
     query: {
-      ids: {
-        values: commentIDs.map(toBase10),
+      match: {
+        link_id: toBase10(threadID),
       },
     },
+    size: 10000,
     _source: [
       'author', 'body', 'created_utc', 'parent_id', 'score',
     ],

src/js/api/reddit/comment.js

@@ -1,105 +1,21 @@
-import { fetchMultiple, jsonMultiple, unique } from 'utils'
+import { chunk, flatten, json } from 'utils'
 import { getAuth } from './auth'
-import { getThread } from './thread'
 
-export const commentLookup = {}
-
-export const getCommentIDs = (
-  subreddit,
-  threadID,
-  commentID = '',
-  results = { // This is ugly i know...
-    ids: [],
-    continueThisThreadIDs: {},
-    morechildrenIDs: {},
-  }
-) => (
-  getComments(subreddit, threadID, commentID)
-    .then(comments => handleComments(comments, subreddit, threadID, commentID, results))
-    .then(() => debug(results))
-    .then(() => unique(results.ids))
-)
-
-const getComments = (subreddit, threadID, commentID) => (
-  getThread(subreddit, threadID, commentID)
-    .then(thread => thread[1].data.children)
-)
-
-const handleComments = (comments, subreddit, threadID, commentID, results) => {
-  // Init ID arrays for this thread of comments ()
-  results.continueThisThreadIDs[commentID] = []
-  results.morechildrenIDs[commentID] = []
-
-  comments.forEach(comment => handleComment(comment, commentID, results))
-
-  return (
-    handleMoreChildren(threadID, commentID, results)
-      .then(() => (
-        Promise.all(results.continueThisThreadIDs[commentID].map(id =>
-          getCommentIDs(subreddit, threadID, id, results)))
-      ))
-  )
-}
-
-const handleComment = (comment, commentID, results) => {
-  // Normal comment
-  if (comment.kind === 't1') {
-    // Has comment replies
-    if (comment.data.replies) {
-      // Handle all replies in the same way
-      comment.data.replies.data.children.forEach(commentReply => handleComment(commentReply, commentID, results))
-      delete comment.data.replies
-    }
-
-    // Add to the return object
-    results.ids.push(comment.data.id)
-    // Store commment in lookup table
-    commentLookup[comment.data.id] = comment.data
-
-  // Special comment
-  } else if (comment.kind === 'more') {
-    // "continue this thread" comment
-    if (comment.data.id === '_') {
-      results.continueThisThreadIDs[commentID].push(comment.data.parent_id.split('_')[1])
-
-    // "Load more"-comment (that is missing some of its children)
-    } else if (comment.data.children.length < comment.data.count) {
-      results.morechildrenIDs[commentID].push(comment.data.children)
-    }
-
-    // Always add the "more"-comments children
-    results.ids.push(...comment.data.children)
-  }
-}
-
-const handleMoreChildren = (threadID, commentID, results) => (
+export const getComments = commentIDs => (
   getAuth()
     .then(auth => (
-      Promise.all(results.morechildrenIDs[commentID].map(idArray =>
-        fetchMultiple(`https://oauth.reddit.com/api/morechildren?link_id=t3_${threadID}&children=`, idArray, auth)))
-        .then(responsesArrays => Promise.all(responsesArrays.map(jsonMultiple)))
-        .then(responsesArrays => {
-          // Reset the array and parse for new morechildren-comments
-          results.morechildrenIDs[commentID] = []
-          responsesArrays.forEach(responses => responses.forEach(response => response.jquery[10][3][0].forEach(comment => {
-            handleComment(comment, commentID, results)
-          })))
-        })
-        .then(() => {
-          if (results.morechildrenIDs[commentID].length !== 0) {
-            return handleMoreChildren(threadID, commentID, results)
-          }
-
-          return Promise.resolve()
-        })
+      Promise.all(chunk(commentIDs, 100).map(ids => fetchComments(ids, auth))).then(flatten)
     ))
 )
 
+export const fetchComments = (commentIDs, auth) => (
+  fetch(`https://oauth.reddit.com/api/info?id=${commentIDs.map(id => `t1_${id}`).join()}`, auth)
+    .then(json)
+    .then(results => results.data.children)
+    .then(commentsData => commentsData.map(commentData => commentData.data))
+)
+
 const debug = results => {
   console.log(JSON.parse(JSON.stringify(results)))
 }
 
-// GET REMAINING COMMENTS
-export const getRemainingComments = commentIDs => {
-
-}

src/js/api/reddit/index.js

@@ -1,5 +1,5 @@
 export { getPost } from './thread'
-export { getCommentIDs } from './comment'
+export { getComments } from './comment'
 //       Status.loading("Getting removed comments...");
 //       HandleIDs.removed();
 //       ThreadHTML.createCommentInfo(Comments.removed.length);

src/js/components/CommentSection.js

@@ -1,7 +1,14 @@
 import React from 'react'
 
+const arrayToLookup = commentList => {
+
+}
+
+
 export default (props) => {
   const { children } = props
+  console.log('COMMENT SECTION RENDERED')
+
   return (
     <div>
       <h1>{props.root}</h1>

src/js/index.js

@@ -7,11 +7,37 @@ import App from 'App'
 
 import '../sass/main.sass'
 
-ReactDOM.render(
-  <Provider store={store}>
-    <App />
-  </Provider>,
-  document.getElementById('app')
-)
+// ReactDOM.render(
+//   <Provider store={store}>
+//     <App />
+//   </Provider>,
+//   document.getElementById('app')
+// )
+import { getComments } from 'api/reddit'
+import { getComments as getAllComments } from 'api/pushshift'
+
+getAllComments('6z1hch')
+  // .then(comments => comments.filter((val, i) => i < 100))
+  .then(comments => {
+    const ids = comments.map(comment => comment.id)
+    // push to state
+    return getComments(ids)
+  })
+//
+// .then(getComments)
+  .then(redditComments => {
+    const removed = []
+    const deleted = []
+
+    redditComments.forEach(comment => {
+      if (comment.body === '[removed]') {
+        removed.push(comment.id)
+      } else if (comment.body === '[deleted]') {
+        deleted.push(comment.id)
+      }
+    })
+
+    console.log('deleted', deleted)
+    console.log('removed', removed)
+  })
 
-// store.dispatch(setStatusLoading('hi'))

src/js/pages/Thread.js

@@ -3,15 +3,13 @@ import Post from 'components/Post'
 import CommentSection from 'components/CommentSection'
 import {
   getPost,
-  getCommentIDs,
+  getComments as getRedditComments,
 } from 'api/reddit'
 import {
   getPost as getRemovedPost,
-  getCommentIDs as getAllCommentIDs,
-  getComments as getRemovedComments,
+  getComments as getPushshiftComments,
 } from 'api/pushshift'
-import { isDeleted } from 'utils'
-import { difference } from '../utils/index'
+import { isDeleted, isRemoved } from 'utils'
 
 export default class Thread extends React.Component {
   constructor(props) {
@@ -19,7 +17,11 @@ export default class Thread extends React.Component {
 
     this.state = {
       post: {},
-      comments: [],
+      pushshiftComments: [],
+      redditComments: [],
+      removed: [],
+      deleted: [],
+      loadingComments: true,
     }
   }
 
@@ -34,26 +36,42 @@ export default class Thread extends React.Component {
           this.setState({ post })
           // Fetch the thread from pushshift if it was deleted/removed
           if (isDeleted(post.selftext)) {
-            // getRemovedPost(threadID)
-            //   .then(removedPost => {
-            //     removedPost.removed = true
-            //     this.setState({ post: removedPost })
-            //   })
+            getRemovedPost(threadID)
+              .then(removedPost => {
+                removedPost.removed = true
+                this.setState({ post: removedPost })
+              })
           }
-        })
-      // Get comment ids from reddit
-        .then(() => getCommentIDs(subreddit, threadID)),
-
+        }),
       // Get comment ids from pushshift
-      getAllCommentIDs(threadID),
+      getPushshiftComments(threadID),
     ])
       .then(results => {
-        const foundIDs = results[0]
-        const allIDs = results[1]
+        const pushshiftComments = results[1]
+        this.setState({ pushshiftComments })
 
-        const removedIDs = difference(allIDs, foundIDs)
-        // Get removed comments from pushshift
-        return getRemovedComments(removedIDs)
+        const ids = pushshiftComments.map(comment => comment.id)
+
+        return getRedditComments(ids)
+      })
+      .then(redditComments => {
+        const removed = []
+        const deleted = []
+
+        redditComments.forEach(comment => {
+          if (isRemoved(comment.body)) {
+            removed.push(comment.id)
+          } else if (isDeleted(comment.body)) {
+            deleted.push(comment.id)
+          }
+        })
+
+        this.setState({
+          removed,
+          deleted,
+          redditComments,
+          loadingComments: false,
+        })
       })
   }
 
@@ -61,7 +79,15 @@ export default class Thread extends React.Component {
     return (
       <div>
         <Post {...this.state.post} />
-        <CommentSection root={this.state.post.id} comments={this.state.comments} />
+        {
+          this.state.loadingComments &&
+          <CommentSection
+            root={this.state.post.id}
+            comments={this.state.pushshiftComments}
+            removed={this.state.removed}
+            deleted={this.state.deleted}
+          />
+        }
       </div>
     )
   }

src/js/utils/index.js

@@ -40,8 +40,11 @@ export const jsonMultiple = responses => Promise.all(responses.map(json))
 export const toBase36 = number => parseInt(number, 10).toString(36)
 export const toBase10 = numberString => parseInt(numberString, 36)
 
-// Reddits way of indicating that something is deleted/removed
-export const isDeleted = testString => testString === '[deleted]'
+// Reddits way of indicating that something is deleted
+export const isDeleted = textBody => textBody === '[deleted]'
+
+// Reddits way of indicating that something is deleted
+export const isRemoved = textBody => textBody === '[removed]'
 
 // Default thumbnails for reddit threads
 export const redditThumbnails = ['self', 'default', 'image', 'nsfw']