Done with pushshift related stuff

src/js/api/pushshift/index.js

@@ -1,11 +1,10 @@
-import { json, toBase10, toBase36 } from 'utils'
+import { json, toBase10, toBase36, chunk, flatten } from 'utils'
 
 const baseURL = 'https://elastic.pushshift.io'
 const threadURL = `${baseURL}/rs/submissions/_search?source=`
 const commentURL = `${baseURL}/rc/comments/_search?source=`
 const commentIDsURL = 'https://api.pushshift.io/reddit/submission/comment_ids/'
 
-
 export const getCommentIDs = threadID => (
   fetch(commentIDsURL + threadID)
     .then(json)
@@ -32,28 +31,35 @@ export const getThread = threadID => {
   )
 }
 
-// getComments (handle more than 100, try 50, 25 etc....)
-export const test = threadID => {
+export const getComments = commentIDs => (
+  Promise.all(chunk(commentIDs, 100).map(fetchComments)).then(flatten)
+)
+
+const fetchComments = commentIDs => {
   const elasticQuery = {
     query: {
-      term: {
-        link_id: toBase10(threadID),
+      ids: {
+        values: commentIDs.map(toBase10),
       },
     },
-    size: 10000,
+    _source: [
+      'author', 'body', 'created_utc', 'parent_id', 'score',
+    ],
   }
 
   return (
     fetch(commentURL + JSON.stringify(elasticQuery))
       .then(json)
-      .then(results => {
-        results.hits.hits.map(result => {
-          result._source.link_id = toBase36(result._source.link_id)
-          result._source.parent_id = toBase36(result._source.parent_id)
-          result._source.id = toBase36(result._id)
+      .then(jsonData => jsonData.hits.hits)
+      .then(comments => comments.map(comment => {
+        comment._source.id = toBase36(comment._id)
 
-          return result._source
-        })
-      })
+        if (!comment._source.parent_id) {
+          console.log('MISSING PARENT ID')
+        }
+
+        comment._source.parent_id = toBase36(comment._source.parent_id)
+        return comment._source
+      }))
   )
 }

src/js/api/reddit/comment.js

@@ -1,4 +1,4 @@
-import { json, fetchMultiple, jsonMultiple, unique } from 'utils'
+import { fetchMultiple, jsonMultiple, unique } from 'utils'
 import { getAuth } from './auth'
 import { getThread } from './thread'
 
@@ -17,7 +17,7 @@ export const getCommentIDs = (
   getComments(subreddit, threadID, commentID)
     .then(comments => handleComments(comments, subreddit, threadID, commentID, results))
     .then(() => debug(results))
-    .then(() => results.ids)
+    .then(() => unique(results.ids))
 )
 
 const getComments = (subreddit, threadID, commentID) => (

src/js/index.js

@@ -16,8 +16,11 @@
 
 // store.dispatch(setStatusLoading('hi'))
 
-import { getCommentIDs } from 'api/reddit'
-import { unique } from 'utils'
+import { getCommentIDs, getThread, getComments } from 'api/pushshift'
 
-getCommentIDs('TwoXChromosomes', '6z1hch')
-  .then(ids => console.log(unique(ids)))
+
+// getCommentIDs('6z1hch')
+
+// getThread('6z1hch').then(console.log)
+getComments(['dmt88s6', 'dmtqxj8', 'dmt1euz']).then(console.log)
+// .then(ids => console.log(unique(ids)))

src/js/utils/index.js

@@ -1,4 +1,4 @@
-import SnuOwnd from 'libraries/snuownd.js'
+import SnuOwnd from 'snuownd'
 
 const markdown = SnuOwnd.getParser()