diff --git a/server.go b/server.go index 9fca51b..1b9342a 100644 --- a/server.go +++ b/server.go @@ -20,11 +20,9 @@ import ( // consts const ( - userAgent = "Javascript:" + appName + ":" + version + "s (by /u/" + userName + ")" - tokenURL = "https://www.reddit.com/api/v1/access_token" - commentIDsURL = "https://api.pushshift.io/reddit/submission/comment_ids/" - commentsURL = "https://api.pushshift.io/reddit/comment/search?ids=" - tmplFolder = "templates/" + userAgent = "Javascript:" + appName + ":" + version + "s (by /u/" + userName + ")" + tokenURL = "https://www.reddit.com/api/v1/access_token" + tmplFolder = "templates/" ) var ( @@ -49,25 +47,6 @@ type tokenResponse struct { Scope string `json:"scope"` } -type pushshiftIDsAPI struct { - Data []string `json:"data"` -} - -type pushshiftCommentsAPI struct { - Data []pushshiftComment `json:"data"` -} - -type pushshiftComment struct { - Author string `json:"author"` - AuthorFlairText string `json:"author_flair_text"` - Body string `json:"body"` - CreatedUTC int32 `json:"created_utc"` - ID string `json:"id"` - IsSubmitter bool `json:"is_submitter"` - ParentID string `json:"parent_id"` - Score int32 `json:"score"` -} - type threadPageData struct { Token string Subreddit string @@ -85,7 +64,6 @@ func main() { http.HandleFunc("/", pageHandler(mainHandler)) http.HandleFunc("/r/", pageHandler(threadHandler)) - http.HandleFunc("/comments/", pageHandler(commentHandler)) // Run locally for debugging/testing if debugMode { @@ -173,65 +151,21 @@ func threadHandler(w http.ResponseWriter, r *http.Request) { return } - resp, _ := client.Get(commentIDsURL + pathParts[4]) - if resp.StatusCode != 200 { - handleError(w, "Trouble getting removed comments from pushshift") - return - } - token, err := getAPIToken() if err != nil { handleError(w, err.Error()) return } - body, err := ioutil.ReadAll(resp.Body) - var dataStructure pushshiftIDsAPI - json.Unmarshal(body, &dataStructure) - data := &threadPageData{ - Token: token, - Subreddit: pathParts[2], - ThreadID: pathParts[4], - CommentIDs: dataStructure.Data, + Token: token, + Subreddit: pathParts[2], + ThreadID: pathParts[4], } renderTemplate(w, "thread", data) } -func commentHandler(w http.ResponseWriter, r *http.Request) { - commentIDs := r.FormValue("c") - if commentIDs == "" { - handleError(w, "Not enough arguments provided") - return - } - - resp, err := client.Get(commentsURL + commentIDs) - - if err != nil { - handleError(w, "Trouble getting removed comments from pushshift1") - return - } - - body, _ := ioutil.ReadAll(resp.Body) - var c pushshiftCommentsAPI - err = json.Unmarshal(body, &c) - - if err != nil { - handleError(w, "Trouble parsing removed comments from pushshift2") - return - } - - jsonString, err := json.Marshal(c) - - if err != nil { - handleError(w, "Trouble parsing removed comments from pushshift3") - return - } - - fmt.Fprintf(w, string(jsonString)) -} - func getAPIToken() (string, error) { req, _ := http.NewRequest("POST", tokenURL, strings.NewReader(tokenData.Encode())) req.Header.Add("User-Agent", userAgent) diff --git a/static/done.png b/static/done.png new file mode 100644 index 0000000..722eef8 Binary files /dev/null and b/static/done.png differ diff --git a/static/loading.gif b/static/loading.gif new file mode 100644 index 0000000..6fd942d Binary files /dev/null and b/static/loading.gif differ diff --git a/static/script.js b/static/script.js index 5a67141..76c47cf 100644 --- a/static/script.js +++ b/static/script.js @@ -1,202 +1,235 @@ "use strict"; const markdown = SnuOwnd.getParser() +const htmlParser = new DOMParser(); -const thread_url = `https://oauth.reddit.com/r/${subreddit}/comments/${thread_id}` -const morechildren_url = `https://oauth.reddit.com/api/morechildren?link_id=t3_${thread_id}&children=` -const single_comment_url = `https://oauth.reddit.com/r/${subreddit}/api/info/?id=t1_` -const removed_comments_url = "/comments/?c=" +const redditThreadURL = `https://oauth.reddit.com/r/${subreddit}/comments/${threadID}` +const redditMorechildrenURL = `https://oauth.reddit.com/api/morechildren?link_id=t3_${threadID}&children=` +const redditSingleCommentURL = `https://oauth.reddit.com/r/${subreddit}/api/info/?id=t1_` +const pushshiftIDsURL = `https://api.pushshift.io/reddit/submission/comment_ids/${threadID}` +const pushshiftCommentsURL = "https://api.pushshift.io/reddit/comment/search?ids=" -const main_div = document.getElementById("main") -const loading_comments = document.getElementById("loading-comments") +const mainDiv = document.getElementById("main") +const loadingText = document.getElementById("loading-text") +const loadingImage = document.getElementById("loading-image") +const loadingImageSrc = "/static/loading.gif" +const doneImageSrc = "/static/done.png" -const comment_ids = [] -const morechildren_ids = [] -const continuethisthread_ids = [] -const max_ids_per_call = 100 -const comment_lookup = new Map() -const comments_to_create = [] -let total_comments +// I actually haven't found a better way of doing this... +// Imgur has images-links with no indication that they are actually images +const imageHosts = ["i.redd.it", "flickr.com", "i.imgur.com", "imgur.com", "m.imgur.com"] -const reddit_init = { +const commentIDs = [] +const morechildrenIDs = [] +const continuethisthreadIDs = [] +const maxIDsPerRequest = 100 +const commentLookup = new Map() +const commentsToCreate = [] + +let totalComments + +const redditInit = { headers: { "Authorization": "bearer " + token } } -load_page() +loadPage() -async function load_page() { - const json = await fetch(thread_url, reddit_init).then(thread => thread.json()) +async function loadPage() { + loadingImage.src = loadingImageSrc + setLoadingText("Loading thread...") - generate_thread(json) - comment_ids.push(...get_comment_ids(json)) + // Get thread from reddit and all comment IDs (inc. removed ones) from pushshift + const requests = [ + fetch(redditThreadURL, redditInit).then(json), + fetch(pushshiftIDsURL).then(json) + ] - await get_morechildren_ids() - - const smaller_threads = await Promise.all(continuethisthread_ids.map(id =>fetch(thread_url + "/_/" + id.split("_")[1], reddit_init))) - const smaller_threads_json = await Promise.all(smaller_threads.map(x => x.json())) - smaller_threads_json.forEach(thread => comment_ids.push(...get_comment_ids(thread))) + // Bottleneck here is allways the reddit api + const [thread, allCommentIDs] = await Promise.all(requests) - const removed_comments = await get_removed_comments() - generate_comments(removed_comments) + generateThread(thread) + + // Get all the comments from the first request + setLoadingText("Loading comments from reddit...") + commentIDs.push(...getCommentIDs(thread)) + + // Recursivly extract all IDs from "show more"-comments + await getMorechildrenIDs() + + // Do same thing for "continue this thread"-comments + const smallerThreads = await Promise.all(continuethisthreadIDs.map(id => fetch(redditThreadURL + "/_/" + id.split("_")[1], redditInit).then(json))) + //const smallerThreadsJson = await Promise.all(smallerThreads.map(x => x.json())) + smallerThreads.forEach(thread => commentIDs.push(...getCommentIDs(thread))) + + setLoadingText("Getting removed comments...") + const removedComments = await getRemovedComments(allCommentIDs.data) + + setLoadingText("Generating comments...") + generateComments(removedComments) + + setLoadingText("") + loadingImage.src = doneImageSrc } // ------------------------------------------------------------------------------ // ----------------------- Functions for getting IDs ---------------------------- // ------------------------------------------------------------------------------ -async function get_removed_comments() { - const ids_diff = all_ids.filter(x => !comment_ids.includes(x)) - generate_comment_info(ids_diff.length) - console.log("All ids :", all_ids) - console.log("Comments ids (dup): ", comment_ids) - console.log("With/without dups: ", comment_ids.length, new Set(comment_ids).size) - console.log("Difference: ", ids_diff) - console.log("Lookup size:", comment_lookup.size) - return fetch_multiple(removed_comments_url, ids_diff, null, ["data"], true) +function getCommentIDs(thread) { + return flattenArray(thread[1].data.children.map(extractIDFromComment)) } -async function get_morechildren_ids() { - //const responses = await Promise.all(morechildren_ids.map(id_array => fetch(morechildren_url + id_array.join(), reddit_init))) - const responses_arrays = await Promise.all(morechildren_ids.map(id_array => fetch_multiple(morechildren_url, id_array, reddit_init))) - const responses = flatten_array(responses_arrays) - morechildren_ids.length = 0 - responses.forEach(extract_morechildren_ids) - - if(morechildren_ids.length !== 0) { - await get_morechildren_ids() - } -} - -function extract_morechildren_ids(comments) { - comments.jquery[14][3][0].forEach(comment => { - if(comment.kind == "more") { - if(comment.data.id === "_") { - console.log("From more, continue: " + comment.data.parent_id) - continuethisthread_ids.push(comment.data.parent_id) - } else { - - const children = comment.data.children - - if(children.length < comment.data.count) { - morechildren_ids.push(children) - } - comment_ids.push(...children) - } - } else { - comment_ids.push(comment.data.id) - comment_lookup.set(comment.data.id, comment.data) - } - }); -} -function get_comment_ids(data) { - return flatten_array(data[1].data.children.map(extract_id_from_comment)) -} - -function extract_id_from_comment(comment) { +function extractIDFromComment(comment) { const data = comment.data // "Show more"-comments - if(comment.kind == "more") { - //console.log(data.id) - //console.log(data.children) - //console.log(data.children.length + "/" + data.count) - //console.log("-----------------------") - - if(data.id === "_") { + if(comment.kind == "more") { + if(data.id === "_") { // = "continue this thread"-comment console.log("From thread countinue: "+ data.parent_id) - continuethisthread_ids.push(data.parent_id) - return [] - } else if(data.children.length < data.count){ - morechildren_ids.push(data.children) - // ???????????????????? maybe wrong, data.id? ytest this - //return [] //----------------------------------- + continuethisthreadIDs.push(data.parent_id) + } else if(data.children.length < data.count){ // "Load more"-comment (that is missing some of its children) + morechildrenIDs.push(data.children) } return data.children } // Normal comment - let replies_ids = [data.id] + const repliesIDs = [data.id] if(data.replies) { - data.replies.data.children.forEach(child => replies_ids.push(...extract_id_from_comment(child))) + data.replies.data.children.forEach(child => repliesIDs.push(...extractIDFromComment(child))) delete data.replies } - comment_lookup.set(data.id, data) - return replies_ids + commentLookup.set(data.id, data) + return repliesIDs } +async function getMorechildrenIDs() { + //const responses = await Promise.all(morechildren_ids.map(id_array => fetch(morechildren_url + id_array.join(), reddit_init))) + const responsesArrays = await Promise.all(morechildrenIDs.map(idArray => fetchMultiple(redditMorechildrenURL, idArray, redditInit))) + const responses = flattenArray(responsesArrays) + morechildrenIDs.length = 0 + responses.forEach(extractMorechildrenIDs) + + if(morechildrenIDs.length !== 0) { + await getMorechildrenIDs() + } +} + +function extractMorechildrenIDs(comments) { + comments.jquery[14][3][0].forEach(comment => { + if(comment.kind == "more") { + if(comment.data.id === "_") { + console.log("From more, continue: " + comment.data.parent_id) + continuethisthreadIDs.push(comment.data.parent_id) + } else { + const children = comment.data.children + + if(children.length < comment.data.count) { + morechildrenIDs.push(children) + } + commentIDs.push(...children) + } + } else { + commentIDs.push(comment.data.id) + commentLookup.set(comment.data.id, comment.data) + } + }); +} + + +async function getRemovedComments(allCommentIDs) { + const idsDiff = allCommentIDs.filter(x => !commentIDs.includes(x)) + generateCommentInfo(idsDiff.length) + console.log("All ids :", allCommentIDs) + console.log("Comments ids (dup): ", commentIDs) + console.log("With/without dups: ", commentIDs.length, new Set(commentIDs).size) + console.log("Difference: ", idsDiff) + console.log("Lookup size:", commentLookup.size) + return fetchMultiple(pushshiftCommentsURL, idsDiff, null, ["data"], true) +} + + // ------------------------------------------------------------------------------ // ----------------------- Comment generating functions ------------------------- // ------------------------------------------------------------------------------ -async function generate_comments(removed_comments) { - const comment_section = document.createElement("div") - comment_section.id = thread_id - main_div.insertBefore(comment_section, loading_comments) +async function generateComments(removedComments) { + const commentSection = generateHTML(`
`) + mainDiv.appendChild(commentSection) - removed_comments.forEach(comment => { + removedComments.forEach(comment => { comment["removed"] = true - comment_lookup.set(comment.id, comment) + commentLookup.set(comment.id, comment) }) - const comments_to_lookup = removed_comments.map(comment => comment.id) - await get_comments_to_generate(comments_to_lookup) - - comments_to_create.sort(function(a, b) { - const a_score = comment_lookup.get(a).score - const b_score = comment_lookup.get(b).score + const commentsToLookup = removedComments.map(comment => comment.id) + await getCommentsToGenerate(commentsToLookup) + + commentsToCreate.sort(function(a, b) { + const aScore = commentLookup.get(a).score + const bScore = commentLookup.get(b).score - return a_score === b_score ? 0 : (a_score > b_score ? 1 : -1) + return aScore === bScore ? 0 : (aScore > bScore ? 1 : -1) }); - create_comments() + createComments() } -async function get_comments_to_generate(comments_to_lookup) { - const new_comments_to_lookup = [] - - comments_to_lookup.forEach(id => { - const parent_id = comment_lookup.get(id).parent_id.split("_")[1] +async function getCommentsToGenerate(commentsToLookup) { + const newCommentsToLookup = [] + const commentsToFetch = [] - if(parent_id === thread_id) {} // Has no parent (is parent of thread) - else if(comments_to_create.includes(parent_id)) {} // Parent already exists, do nothing - else if(new_comments_to_lookup.includes(parent_id)) {} // Parent already exists (this iteration) - else if(comment_lookup.has(parent_id)) { - new_comments_to_lookup.push(parent_id) + commentsToLookup.forEach(id => { + const parentID = commentLookup.get(id).parent_id.split("_")[1] + + if(parentID === threadID) {} // Has no parent (is parent of thread) + else if(commentsToCreate.includes(parentID)) {} // Parent already exists, do nothing + else if(newCommentsToLookup.includes(parentID)) {} // Parent already exists (this iteration) + else if(commentLookup.has(parentID)) { + newCommentsToLookup.push(parentID) } else{ - console.error("Comment doesn't exists, HALP") - console.log(id) - console.log(parent_id) + console.error("Comment doesn't exists, but lets try it anyway :D") + console.log("ID: " + id) + console.log("Parent: " + parent_id) + + commentsToFetch.push(parentID) } - comments_to_create.push(id) + commentsToCreate.push(id) }) - if(new_comments_to_lookup.length !== 0) { - await get_comments_to_generate(new_comments_to_lookup) + if(commentsToFetch.length !== 0) { + //await + console.log("fuck it") + } + + if(newCommentsToLookup.length !== 0) { + await getCommentsToGenerate(newCommentsToLookup) } } -function create_comments() { - const created_comment_ids = [thread_id] - let id, parent_id - let did_something +function createComments() { + const createdCommentIDs = [threadID] + let id, parentID + let didSomething - while(comments_to_create.length > 0) { - did_something = false + while(commentsToCreate.length > 0) { + didSomething = false - for(let i = comments_to_create.length -1; i >= 0; i--) { - id = comments_to_create[i] - parent_id = comment_lookup.get(id).parent_id.split("_")[1] + for(let i = commentsToCreate.length -1; i >= 0; i--) { + id = commentsToCreate[i] + parentID = commentLookup.get(id).parent_id.split("_")[1] - if(created_comment_ids.includes(parent_id)) { - document.getElementById(parent_id).appendChild(create_comment(comment_lookup.get(id))) - created_comment_ids.push(id) - comments_to_create.splice(i, 1) - did_something = true + if(createdCommentIDs.includes(parentID)) { + document.getElementById(parentID).appendChild(createComment(commentLookup.get(id))) + createdCommentIDs.push(id) + commentsToCreate.splice(i, 1) + didSomething = true } } - if(!did_something) { + // Fail safe (parents missing for all comments left, should happend but oh well :D) + if(!didSomething) { console.error("Didn't generate all comments correctly") break } @@ -206,35 +239,40 @@ function create_comments() { // ------------------------- HTML-generating functions -------------------------- // ------------------------------------------------------------------------------ -function generate_thread(data) { +function generateThread(data) { const thread = data[0].data.children[0].data - const thread_div = document.createElement("div") - thread_div.id = "thread" - thread_div.innerHTML = ` -Loading...
+