diff --git a/server.go b/server.go index e0c9abc..158870c 100644 --- a/server.go +++ b/server.go @@ -21,7 +21,6 @@ const ( commentIDsURL = "https://api.pushshift.io/reddit/submission/comment_ids/" commentsURL = "https://api.pushshift.io/reddit/comment/search?ids=" tmplFolder = "templates/" - errorHTML = "

Error: %s

" ) var ( @@ -50,7 +49,7 @@ type pushshiftAPI struct { Data []string `json:"data"` } -type pageData struct { +type threadPageData struct { Token string Subreddit string ThreadID string @@ -88,7 +87,7 @@ func initLogging() { errorLog = log.New(errorFile, "", log.Ldate|log.Ltime|log.Lshortfile) } -// Wrap handle function for the access log +// Wrap page handle function for the access log func pageHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { accessLog.Println(r.URL.Path) @@ -130,7 +129,7 @@ func threadHandler(w http.ResponseWriter, r *http.Request) { var dataStructure pushshiftAPI json.Unmarshal(body, &dataStructure) - data := &pageData{ + data := &threadPageData{ Token: token, Subreddit: pathParts[2], ThreadID: pathParts[4], diff --git a/static/favicon (copy).ico b/static/favicon.ico similarity index 100% rename from static/favicon (copy).ico rename to static/favicon.ico diff --git a/static/script.js b/static/script.js index bdfbe96..f57c91d 100644 --- a/static/script.js +++ b/static/script.js @@ -26,6 +26,8 @@ load_page() async function load_page() { const json = await fetch(thread_url, reddit_init).then(thread => thread.json()) + //const json = await fetch(morechildren_url + "dmh2rmf,dmglplf,dmgq1zv,dmgu4lb", reddit_init).then(thread => thread.json()) + //console.log(json) generate_thread(json) comment_ids.push(...get_comment_ids(json)) @@ -47,10 +49,11 @@ async function load_page() { async function get_removed_comments() { const ids_diff = all_ids.filter(x => !comment_ids.includes(x)) generate_comment_info(ids_diff.length) - console.log(all_ids) - console.log(comment_ids) - console.log(comment_ids.length, new Set(comment_ids).size) - console.log(ids_diff) + console.log("All ids :", all_ids) + console.log("Comments ids (dup): ", comment_ids) + console.log("With/without dups: ", comment_ids.length, new Set(comment_ids).size) + console.log("Difference: ", ids_diff) + console.log("Lookup size:", comment_lookup.size) return fetch_multiple(removed_comments_url, ids_diff, ["data"], true) } @@ -59,8 +62,9 @@ async function get_continuethisthread_ids() { } async function get_morechildren_ids() { - console.log("asdf") - const responses = await Promise.all(split_array(morechildren_ids, max_ids_per_call).map(ids => fetch(morechildren_url + ids.join(), reddit_init))) + //const responses = await Promise.all(split_array(morechildren_ids, max_ids_per_call).map(ids => fetch(morechildren_url + ids.join(), reddit_init))) + const responses = await Promise.all(morechildren_ids.map(id_array => fetch(morechildren_url + id_array.join(), reddit_init))) + const responses_json = await Promise.all(responses.map(x => x.json())) morechildren_ids.length = 0 responses_json.forEach(extract_morechildren_ids) @@ -76,7 +80,7 @@ function extract_morechildren_ids(comments) { const children = comment.data.children if(children.length < comment.data.count) { - morechildren_ids.push(...children) + morechildren_ids.push(children) } comment_ids.push(...children) @@ -94,19 +98,16 @@ function extract_id_from_comment(comment) { const data = comment.data // "Show more"-comments if(comment.kind == "more") { - console.log(data.id) - console.log(data.name) - console.log(data.parent_id) - console.log(data.children) - console.log(data.children.length + "/" + data.count) - console.log("-----------------------") + //console.log(data.id) + //console.log(data.children) + //console.log(data.children.length + "/" + data.count) + //console.log("-----------------------") if(data.id === "_") { continuethisthread_ids.push(data.parent_id) return [] - } - if(data.children.length < data.count){ - morechildren_ids.push(...data.children) + } else if(data.children.length < data.count){ + morechildren_ids.push(data.children) // ???????????????????? maybe wrong, data.id? ytest this //return [] //----------------------------------- } @@ -136,7 +137,6 @@ async function generate_comments(removed_comments) { removed_comments.forEach(comment => { comment["removed"] = true comment_lookup.set(comment.id, comment) - console.log(comment.id) }) const comments_to_lookup = removed_comments.map(comment => comment.id) @@ -280,6 +280,7 @@ async function fetch_multiple(url, data, json_walkdown=[], flattening=false) { // ------------------------------------------------------------------------------ // ---------------------- Other less interesting functions ---------------------- // ------------------------------------------------------------------------------ + function split_array(array, size) { const array_split = [] diff --git a/templates/footer.html b/templates/footer.html index 0d43988..92f6532 100644 --- a/templates/footer.html +++ b/templates/footer.html @@ -1,4 +1,3 @@ - - \ No newline at end of file + \ No newline at end of file diff --git a/templates/frontpage.html b/templates/frontpage.html index d3ea553..de84c3b 100644 --- a/templates/frontpage.html +++ b/templates/frontpage.html @@ -18,6 +18,7 @@

Acknowledgement

diff --git a/templates/thread.html b/templates/thread.html index 047d49c..b4ae645 100644 --- a/templates/thread.html +++ b/templates/thread.html @@ -6,4 +6,5 @@ const subreddit = "{{ .Subreddit }}"; const all_ids = {{ .CommentIDs }}; - \ No newline at end of file + + \ No newline at end of file