diff --git a/app.py b/app.py deleted file mode 100644 index df61dd8..0000000 --- a/app.py +++ /dev/null @@ -1,86 +0,0 @@ -from flask import Flask, abort, request, render_template -import urllib.parse -import requests -import requests.auth -import datetime - -import json - -APP_NAME = 'TEST' -USERNAME = 'Jubbeart' -VERSION = '0.1' - -CLIENT_ID = '' -CLIENT_SECRET = '' -USER_AGENT = 'Javascript:{}:v{} (by /u/{})'.format(APP_NAME, VERSION, USERNAME) - -TOKEN_URL = 'https://www.reddit.com/api/v1/access_token' -API_URL = 'https://oauth.reddit.com/' -COMMENT_IDS_URL = 'https://api.pushshift.io/reddit/submission/comment_ids/{}' -COMMENTS_URL = 'https://api.pushshift.io/reddit/comment/search?ids={}' - -FILTER_BODY = [ - 'title', 'author', 'num_comments', - 'subreddit', 'score', 'upvote_ratio', - 'created_utc', 'locked', 'link_flair_text', - 'is_self', 'url', 'selftext_html', - 'id' -] - -FILTER_COMMENT = [ - 'score', 'author', 'created_utc', - 'body_html', 'id', 'controversiality', - - 'collapsed_reason', 'collapsed', -] - -app = Flask(__name__) - -@app.route('/') -def homepage(): - return render_template('frontpage.html') - - -@app.route('/r//comments/') -def thread(subreddit, thread_id): - thread_id = thread_id.split('/')[0] - comment_ids = requests.get(COMMENT_IDS_URL.format(thread_id)).json()["data"] - token = get_token() - - return render_template( - 'thread.html', - token=token, - comment_ids=comment_ids, - thread_id=thread_id, - subreddit=subreddit - ) - -@app.route('/comments/') -def comments(): - ids = None - - try: - ids = request.args.get('c') - except: - return '' - - if ids: - return requests.get(COMMENTS_URL.format(ids)).content - else: - return '' - - -def get_token(old_token=None): - post_data = {'grant_type': 'client_credentials'} - headers = {'User-Agent': USER_AGENT} - client_auth = requests.auth.HTTPBasicAuth(CLIENT_ID, CLIENT_SECRET) - response = requests.post( - TOKEN_URL, - auth=client_auth, - data=post_data, - headers=headers - ) - return response.json()['access_token'] - -if __name__ == '__main__': - app.run(debug=True, port=65010) diff --git a/old_code b/old_code deleted file mode 100644 index 8f83c1e..0000000 --- a/old_code +++ /dev/null @@ -1,176 +0,0 @@ - -/*function create_parent_comments() { - var comment_section = document.getElementById("comments"); - var comments = comment_section.children; - var parent_id; - var comment; - var created_comments = []; - var done = true; - - for(var i = comments.length - 1; i >= 0; i--) { - //console.log(comments[i]); - - parent_id = comment_lookup[comments[i].id].parent_id; - - if(parent_id.includes("_")) { - parent_id = parent_id.split("_")[1]; - } - - // Done with this comment - if(parent_id == thread_id) { - //console.log(i, "done"); - } - // Check if parent already exists - else if(comments_on_display.has(parent_id)) { - document.getElementById(parent_id).appendChild(comments[i]); - //console.log(i, "comment already exists"); - done = false; - } - // comment_data exists but parent comment hasn't been created yet - else if(comment_lookup.hasOwnProperty(parent_id)) { - //created_comments.push(create_comment(comments_data[parent_id])); - comment_section.appendChild(create_comment(comment_lookup[parent_id], false)); - //console.log(i, " comment data exists"); - //done = false; - } - // parent comment doesn't exists - else { - get(single_comment_url + parent_id, function(data){ - var comment = create_comment(JSON.parse(data).data.children[0].data, false); - comment_section.appendChild(); - add_parent_comments(); - console.log(parent_id, " created via /api/info ???"); - }); - - console.log(i, " doesn't exist"); - } - } - - if(!done) { - add_parent_comments(); - } else { - fix_background_color(); - sort_comments(); - } - -} -*/ - - - - - - - - - - - - - - -/* -function get_comments(json) { - var comments = []; - var comments_json = json[1].data.children; - - for(var i = 0; i < comments_json.length; i++) { - get_comment_data(comments_json[i], comments); - } - - return comments; -} - - -var stripped_comment_tags = ["author_flair_css_class", "approved_by", -"approved_at_utc", "archived", "author_flair_text", -"body_html", "banned_at_utc", "banned_by", -"can_gild", "can_mod_post", "created", "collapsed", "collapsed_reason", -"downs", "distinguished", "gilded", -"edited", "likes", "link_id", "name", "mod_reports", "num_reports", -"removal_reason", "report_reasons", -"saved", "subreddit", "subreddit_type", "subreddit_name_prefixed", "subreddit_id", -"ups", "user_reports"]; - - - - -// Strips unnecessary tags from comment, adds to comment array -// Do this recursivly to all replies -// Params comment:JSON object, comments: array -function get_comment_data(comment, comments) { - if(comment.kind !== "t1") { - alert(comment.kind); - return; - } - - var comment_data = comment.data; - - for(var i = 0; i < stripped_comment_tags.length; i++) { - delete comment_data[stripped_comment_tags[i]]; - } - - if(comment_data.replies) { - for(var i = 0; i < comment_data.replies.data.children.length; i++) { - get_comment_data(comment_data.replies.data.children[i], comments); - } - } - - if(comment_data.parent_id.indexOf("_") !== -1) { - comment_data.parent_id = comment_data.parent_id.split("_")[1]; - } - - delete comment_data["replies"]; - comments.push(comment_data); -} -*/ - - -old_code: - - - """headers = {'Authorization': 'bearer ' + token, 'User-Agent': USER_AGENT} - url = API_URL + THREAD_PATH.format(subreddit, thread_id) - - response = requests.get(url, headers=headers) - thread_json = None - - try: - thread_json = response.json() - except: - return 'Error parsing the thread' - - thread = { - 'body': get_thread_body(thread_json), - 'comments': get_thread_comments(thread_json) - } - - return str(get_thread_comments(thread_json)) - #return str(get_thread_comments(thread_json)) + '/' + str(thread['body']['num_comments']) - -def get_thread_body(thread_json): - body_json = thread_json[0]['data']['children'][0]['data'] - filtered_body = {k:v for k,v in body_json.items() if k in FILTER_BODY} - return filtered_body - -def get_thread_comments(thread_json): - comments_json = thread_json[1]['data']['children'] - filtered_comments = [] - - for comment_json in comments_json: - comment_data = get_comment_data(comment_json) - filtered_comments.append(comment_data) - - return filtered_comments - -def get_comment_data(comment): - comment_data = comment['data'] - filtered_comment = {k:v for k,v in comment_data.items() if k in FILTER_COMMENT} - filtered_comment['replies'] = [] - - if comment_data['replies']: - for child_comment in comment_data['replies']['data']['children']: - filtered_comment['replies'].append(get_comment_data(child_comment)) - - return filtered_comment - """ diff --git a/secret.go b/secret.go new file mode 100644 index 0000000..7858ec0 --- /dev/null +++ b/secret.go @@ -0,0 +1,9 @@ +package main + +const ( + appName = "" + userName = "" + clientID = "" + clientSecret = "" + version = "" +) diff --git a/server.go b/server.go new file mode 100644 index 0000000..e0c9abc --- /dev/null +++ b/server.go @@ -0,0 +1,180 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "html/template" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// consts +const ( + userAgent = "Javascript:" + appName + ":" + version + "s (by /u/" + userName + ")" + tokenURL = "https://www.reddit.com/api/v1/access_token" + commentIDsURL = "https://api.pushshift.io/reddit/submission/comment_ids/" + commentsURL = "https://api.pushshift.io/reddit/comment/search?ids=" + tmplFolder = "templates/" + errorHTML = "

Error: %s

" +) + +var ( + client = &http.Client{Timeout: time.Second * 10} + tokenData = url.Values{"grant_type": []string{"client_credentials"}} + templates = template.Must(template.ParseFiles( + tmplFolder+"header.html", + tmplFolder+"footer.html", + tmplFolder+"thread.html", + tmplFolder+"frontpage.html", + tmplFolder+"error.html", + )) + + accessLog *log.Logger + errorLog *log.Logger +) + +type tokenResponse struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` + Scope string `json:"scope"` +} + +type pushshiftAPI struct { + Data []string `json:"data"` +} + +type pageData struct { + Token string + Subreddit string + ThreadID string + CommentIDs []string +} + +func main() { + initLogging() + accessLog.Println("Staring server") + /* + s := &http.Server{ + Addr: ":8080", + Handler: myHandler, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + }*/ + + // Only serve local static files for debugging, otherwise use cdn.rawgit.com + fs := http.FileServer(http.Dir("static")) + http.Handle("/static/", http.StripPrefix("/static/", fs)) + + http.HandleFunc("/", pageHandler(mainHandler)) + http.HandleFunc("/r/", pageHandler(threadHandler)) + http.HandleFunc("/comments/", pageHandler(commentHandler)) + + http.ListenAndServe(":9000", nil) +} + +func initLogging() { + accessFile, _ := os.OpenFile("access.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + errorFile, _ := os.OpenFile("error.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + + accessLog = log.New(accessFile, "", log.Ldate|log.Ltime|log.Lshortfile) + errorLog = log.New(errorFile, "", log.Ldate|log.Ltime|log.Lshortfile) +} + +// Wrap handle function for the access log +func pageHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + accessLog.Println(r.URL.Path) + fn(w, r) + } +} + +func renderTemplate(w http.ResponseWriter, pageName string, data interface{}) { + templates.ExecuteTemplate(w, "header.html", nil) + templates.ExecuteTemplate(w, pageName+".html", data) + templates.ExecuteTemplate(w, "footer.html", nil) +} + +func mainHandler(w http.ResponseWriter, r *http.Request) { + renderTemplate(w, "frontpage", nil) +} + +func threadHandler(w http.ResponseWriter, r *http.Request) { + pathParts := strings.Split(r.URL.Path, "/") + + if len(pathParts) < 5 { + handleError(w, "Missing necessary parts of the URL") + return + } + + resp, _ := client.Get(commentIDsURL + pathParts[4]) + if resp.StatusCode != 200 { + handleError(w, "Trouble getting removed comments from pushshift") + return + } + + token, err := getAPIToken() + if err != nil { + handleError(w, err.Error()) + return + } + + body, err := ioutil.ReadAll(resp.Body) + var dataStructure pushshiftAPI + json.Unmarshal(body, &dataStructure) + + data := &pageData{ + Token: token, + Subreddit: pathParts[2], + ThreadID: pathParts[4], + CommentIDs: dataStructure.Data, + } + + renderTemplate(w, "thread", data) +} + +func handleError(w http.ResponseWriter, msg string) { + renderTemplate(w, "error", msg) + errorLog.Println(msg) +} + +func commentHandler(w http.ResponseWriter, r *http.Request) { + commentIDs := r.FormValue("c") + if commentIDs == "" { + handleError(w, "Not enough arguments provided") + return + } + + resp, err := client.Get(commentsURL + commentIDs) + + if err != nil { + handleError(w, "Trouble getting removed comments from pushshift") + return + } + + body, _ := ioutil.ReadAll(resp.Body) + fmt.Fprintf(w, string(body)) +} + +func getAPIToken() (string, error) { + req, _ := http.NewRequest("POST", tokenURL, strings.NewReader(tokenData.Encode())) + req.Header.Add("User-Agent", userAgent) + req.SetBasicAuth(clientID, clientSecret) + + resp, err := client.Do(req) + if err != nil { + return "", errors.New("Coulnd't get API token from Reddit") + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + var r tokenResponse + json.Unmarshal(body, &r) + return r.AccessToken, nil +} diff --git a/static/favicon (copy).ico b/static/favicon (copy).ico new file mode 100644 index 0000000..2a2117a Binary files /dev/null and b/static/favicon (copy).ico differ diff --git a/static/functions.js b/static/functions.js deleted file mode 100644 index e69de29..0000000 diff --git a/static/script.js b/static/script.js index 30d6f78..bdfbe96 100644 --- a/static/script.js +++ b/static/script.js @@ -40,6 +40,10 @@ async function load_page() { generate_comments(removed_comments) } +// ------------------------------------------------------------------------------ +// ----------------------- Functions for getting IDs ---------------------------- +// ------------------------------------------------------------------------------ + async function get_removed_comments() { const ids_diff = all_ids.filter(x => !comment_ids.includes(x)) generate_comment_info(ids_diff.length) @@ -120,9 +124,9 @@ function extract_id_from_comment(comment) { return replies_ids } -// ------------------------------------------------------------------------------------- -// ----------------- Comment generating functions --------------------- -// ------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------ +// ----------------------- Comment generating functions ------------------------- +// ------------------------------------------------------------------------------ async function generate_comments(removed_comments) { const comment_section = document.createElement("div") @@ -199,10 +203,9 @@ function create_comments() { } } } - -// ------------------------------------------------------------------------------------- -// --------------------- HTML-generating functions ---------------------- -// ------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------ +// ------------------------- HTML-generating functions -------------------------- +// ------------------------------------------------------------------------------ function generate_thread(data) { const thread = data[0].data.children[0].data @@ -260,9 +263,10 @@ function generate_comment_info(removed_comments) { ` main_div.insertBefore(comment_info, loading_comments) } -// ------------------------------------------------------------------------------------- + +// ------------------------------------------------------------------------------ // ----------------------------- AJAX-functions --------------------------------- -// ------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------ async function fetch_multiple(url, data, json_walkdown=[], flattening=false) { const responses = await Promise.all(split_array(data, max_ids_per_call).map(single_request_data => fetch(url + single_request_data.join()))) @@ -272,10 +276,10 @@ async function fetch_multiple(url, data, json_walkdown=[], flattening=false) { }) return flattening ? flatten_array(json) : json } -// ------------------------------------------------------------------------------------- -// ---------------- Other less interesting functions --------------------- -// ------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------ +// ---------------------- Other less interesting functions ---------------------- +// ------------------------------------------------------------------------------ function split_array(array, size) { const array_split = [] diff --git a/static/style.css b/static/style.css index 983214a..ca8255f 100644 --- a/static/style.css +++ b/static/style.css @@ -42,17 +42,17 @@ a:hover { } #header { - background-color: #181818; + background-color: #ca302c;/*#19171c;/*#181818*/ padding: 20px 10px 10px; } #header a { - color: #ddd; + color: #fff;/*#ddd;*/ transition: color 0.3s; } #header a:hover { - color: #ca302c; + color: #FF8B88;/*#ca302c*/; text-decoration: none; } @@ -66,6 +66,32 @@ a:hover { margin: 15px; } +#main-box { + margin: 0 auto; + padding: 20px; + border-radius: 5px; + color: #fff; +} + +.error-box { + max-width: 500px; + background-color: #ca302c; +} + +.frontpage-box { + max-width: 800px; + background-color: #161616; + +} + +.frontpage-box p { + margin: 15px; +} + +.error-title { + margin: 0 0 15px 0; +} + #thread { display: flex; } diff --git a/templates/error.html b/templates/error.html new file mode 100644 index 0000000..eca8550 --- /dev/null +++ b/templates/error.html @@ -0,0 +1,4 @@ +
+

Error 500: Internal server error

+

Details: {{.}}

+
\ No newline at end of file diff --git a/templates/footer.html b/templates/footer.html index f82581f..0d43988 100644 --- a/templates/footer.html +++ b/templates/footer.html @@ -1,6 +1,4 @@ -
- - - + + \ No newline at end of file diff --git a/templates/frontpage.html b/templates/frontpage.html index cc1d496..d3ea553 100644 --- a/templates/frontpage.html +++ b/templates/frontpage.html @@ -1,7 +1,41 @@ -{% extends "main.html" %} -{% block body %} - -frontpage - -{% endblock %} - +
+

About

+

+ This is a site for viewing removed comments from Reddit. + It was created by Jesper Wrang and uses Jason Baumgartner service for getting removed comments.

+

+ How to use: go to any Reddit thread and replace the reddit in the URL with removeddit +
E.g. + + https://www.removeddit.com/r/changelog/comments/6xfyfg/ + +

+

+ Note that this site is in it's early stages of development, meaning the site is unstable and will contain bugs. + If you find something that doesn't work please contact me at:
jesper (at) removeddit.com +

+

TODO

+
    +
  • Collapsing comments
    • +
  • Getting removed selftext of thread
    • +
+

Acknowledgement

+

+ Jason Baumgartner - for making all this possible with his amazing API +

+

+ Scott McClaugherty - for his reddit markdown parser +

+

+ /r/redditdev - for helping with the Reddit API +

+

Links/Contact

+

For feedback and bug reports: +

    +
  • email: jesper (at) removeddit.com
    • +
  • reddit: /u/JubbeArt
    • +
+

+ Github for this project +

+
\ No newline at end of file diff --git a/templates/header.html b/templates/header.html index aa9d634..ccfff2c 100644 --- a/templates/header.html +++ b/templates/header.html @@ -1,15 +1,14 @@ - -Removeddit - - - + Removeddit + + + + - - -
+ +
\ No newline at end of file diff --git a/templates/main.html b/templates/main.html deleted file mode 100644 index 0a1a98b..0000000 --- a/templates/main.html +++ /dev/null @@ -1,4 +0,0 @@ -{% include "header.html" %} -{% block body%} -{% endblock %} -{% include "footer.html" %} diff --git a/templates/thread.html b/templates/thread.html index 38b8c2a..047d49c 100644 --- a/templates/thread.html +++ b/templates/thread.html @@ -1,16 +1,9 @@ -{% extends "main.html" %} -{% block body %}
- - - - + - -{% endblock %} - + \ No newline at end of file