Changed how morechildren is being called

server.go

@@ -21,7 +21,6 @@ const (
 	commentIDsURL = "https://api.pushshift.io/reddit/submission/comment_ids/"
 	commentsURL   = "https://api.pushshift.io/reddit/comment/search?ids="
 	tmplFolder    = "templates/"
-	errorHTML     = "<h3 style=\"colorError;font-family: verdana, arial, helvetica, sans-serif;\">Error: %s</h3>"
 )
 
 var (
@@ -50,7 +49,7 @@ type pushshiftAPI struct {
 	Data []string `json:"data"`
 }
 
-type pageData struct {
+type threadPageData struct {
 	Token      string
 	Subreddit  string
 	ThreadID   string
@@ -88,7 +87,7 @@ func initLogging() {
 	errorLog = log.New(errorFile, "", log.Ldate|log.Ltime|log.Lshortfile)
 }
 
-// Wrap handle function for the access log
+// Wrap page handle function for the access log
 func pageHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		accessLog.Println(r.URL.Path)
@@ -130,7 +129,7 @@ func threadHandler(w http.ResponseWriter, r *http.Request) {
 	var dataStructure pushshiftAPI
 	json.Unmarshal(body, &dataStructure)
 
-	data := &pageData{
+	data := &threadPageData{
 		Token:      token,
 		Subreddit:  pathParts[2],
 		ThreadID:   pathParts[4],

static/script.js

@@ -26,6 +26,8 @@ load_page()
 
 async function load_page() {
 	const json = await fetch(thread_url, reddit_init).then(thread => thread.json())
+	//const json = await fetch(morechildren_url + "dmh2rmf,dmglplf,dmgq1zv,dmgu4lb", reddit_init).then(thread => thread.json())
+	//console.log(json)
 
 	generate_thread(json)
 	comment_ids.push(...get_comment_ids(json))
@@ -47,10 +49,11 @@ async function load_page() {
 async function get_removed_comments() {
 	const ids_diff = all_ids.filter(x => !comment_ids.includes(x))
 	generate_comment_info(ids_diff.length)
-	console.log(all_ids)
-	console.log(comment_ids)
-	console.log(comment_ids.length, new Set(comment_ids).size)
-	console.log(ids_diff)
+	console.log("All ids :", all_ids)
+	console.log("Comments ids (dup): ", comment_ids)
+	console.log("With/without dups: ", comment_ids.length, new Set(comment_ids).size)
+	console.log("Difference: ", ids_diff)
+	console.log("Lookup size:", comment_lookup.size)
 	return fetch_multiple(removed_comments_url, ids_diff, ["data"], true)
 }
 
@@ -59,8 +62,9 @@ async function get_continuethisthread_ids() {
 }
 
 async function get_morechildren_ids() {
-	console.log("asdf")
-	const responses = await Promise.all(split_array(morechildren_ids, max_ids_per_call).map(ids => fetch(morechildren_url + ids.join(), reddit_init)))
+	//const responses = await Promise.all(split_array(morechildren_ids, max_ids_per_call).map(ids => fetch(morechildren_url + ids.join(), reddit_init)))
+	const responses = await Promise.all(morechildren_ids.map(id_array => fetch(morechildren_url + id_array.join(), reddit_init)))
+	
 	const responses_json = await Promise.all(responses.map(x => x.json()))
 	morechildren_ids.length = 0
 	responses_json.forEach(extract_morechildren_ids)
@@ -76,7 +80,7 @@ function extract_morechildren_ids(comments) {
 			const children = comment.data.children
 
 			if(children.length < comment.data.count) {
-				morechildren_ids.push(...children)			
+				morechildren_ids.push(children)			
 			} 
 			comment_ids.push(...children)
 			
@@ -94,19 +98,16 @@ function extract_id_from_comment(comment) {
 	const data = comment.data
 	// "Show more"-comments
 	if(comment.kind == "more") {	
-		console.log(data.id)
-		console.log(data.name)
-		console.log(data.parent_id)
-		console.log(data.children)
-		console.log(data.children.length + "/" + data.count)
-		console.log("-----------------------")	
+		//console.log(data.id)
+		//console.log(data.children)
+		//console.log(data.children.length + "/" + data.count)
+		//console.log("-----------------------")	
 		
 		if(data.id === "_") {
 			continuethisthread_ids.push(data.parent_id)
 			return []
-		}		
-		if(data.children.length < data.count){
-			morechildren_ids.push(...data.children)
+		} else if(data.children.length < data.count){
+			morechildren_ids.push(data.children)
 			// ???????????????????? maybe wrong, data.id? ytest this
 			//return [] //-----------------------------------
 		}
@@ -136,7 +137,6 @@ async function generate_comments(removed_comments) {
 	removed_comments.forEach(comment => {
 		comment["removed"] = true
 		comment_lookup.set(comment.id, comment)
-		console.log(comment.id)
 	})
 
 	const comments_to_lookup = removed_comments.map(comment => comment.id)	
@@ -280,6 +280,7 @@ async function fetch_multiple(url, data, json_walkdown=[], flattening=false) {
 // ------------------------------------------------------------------------------
 // ---------------------- Other less interesting functions ----------------------
 // ------------------------------------------------------------------------------
+
 function split_array(array, size) {
 	const array_split = []
 	

templates/footer.html

@@ -1,4 +1,3 @@
 	</div>
 </body>
-</html>
-	
+</html>

templates/frontpage.html

@@ -18,6 +18,7 @@
 	<ul>
 		<li>Collapsing comments</li>
 		<li>Getting removed selftext of thread</li>
+		<li>Getting "continue this thread"-comments to work</li>
 	</ul>
 	<h2 style="color:#1B767A">Acknowledgement</h2>
 		<p>

templates/thread.html

@@ -6,4 +6,5 @@
 	const subreddit = "{{ .Subreddit }}";
 	const all_ids = {{ .CommentIDs }};
 </script>
-<script src="https://cdn.rawgit.com/JubbeArt/removeddit/5a25b161/static/script.js"></script>
+<script src="/static/script.js"></script>
+<!--https://cdn.rawgit.com/JubbeArt/removeddit/5a25b161/static/script.js-->