Changed backend from Flask to Go

This commit is contained in:
Jesper Wrang 2017-09-03 17:52:38 +02:00
parent 5a25b16127
commit cf3bfda80e
14 changed files with 296 additions and 315 deletions

86
app.py
View file

@ -1,86 +0,0 @@
from flask import Flask, abort, request, render_template
import urllib.parse
import requests
import requests.auth
import datetime
import json
APP_NAME = 'TEST'
USERNAME = 'Jubbeart'
VERSION = '0.1'
CLIENT_ID = ''
CLIENT_SECRET = ''
USER_AGENT = 'Javascript:{}:v{} (by /u/{})'.format(APP_NAME, VERSION, USERNAME)
TOKEN_URL = 'https://www.reddit.com/api/v1/access_token'
API_URL = 'https://oauth.reddit.com/'
COMMENT_IDS_URL = 'https://api.pushshift.io/reddit/submission/comment_ids/{}'
COMMENTS_URL = 'https://api.pushshift.io/reddit/comment/search?ids={}'
FILTER_BODY = [
'title', 'author', 'num_comments',
'subreddit', 'score', 'upvote_ratio',
'created_utc', 'locked', 'link_flair_text',
'is_self', 'url', 'selftext_html',
'id'
]
FILTER_COMMENT = [
'score', 'author', 'created_utc',
'body_html', 'id', 'controversiality',
'collapsed_reason', 'collapsed',
]
app = Flask(__name__)
@app.route('/')
def homepage():
return render_template('frontpage.html')
@app.route('/r/<string:subreddit>/comments/<path:thread_id>')
def thread(subreddit, thread_id):
thread_id = thread_id.split('/')[0]
comment_ids = requests.get(COMMENT_IDS_URL.format(thread_id)).json()["data"]
token = get_token()
return render_template(
'thread.html',
token=token,
comment_ids=comment_ids,
thread_id=thread_id,
subreddit=subreddit
)
@app.route('/comments/')
def comments():
ids = None
try:
ids = request.args.get('c')
except:
return ''
if ids:
return requests.get(COMMENTS_URL.format(ids)).content
else:
return ''
def get_token(old_token=None):
post_data = {'grant_type': 'client_credentials'}
headers = {'User-Agent': USER_AGENT}
client_auth = requests.auth.HTTPBasicAuth(CLIENT_ID, CLIENT_SECRET)
response = requests.post(
TOKEN_URL,
auth=client_auth,
data=post_data,
headers=headers
)
return response.json()['access_token']
if __name__ == '__main__':
app.run(debug=True, port=65010)

176
old_code
View file

@ -1,176 +0,0 @@
/*function create_parent_comments() {
var comment_section = document.getElementById("comments");
var comments = comment_section.children;
var parent_id;
var comment;
var created_comments = [];
var done = true;
for(var i = comments.length - 1; i >= 0; i--) {
//console.log(comments[i]);
parent_id = comment_lookup[comments[i].id].parent_id;
if(parent_id.includes("_")) {
parent_id = parent_id.split("_")[1];
}
// Done with this comment
if(parent_id == thread_id) {
//console.log(i, "done");
}
// Check if parent already exists
else if(comments_on_display.has(parent_id)) {
document.getElementById(parent_id).appendChild(comments[i]);
//console.log(i, "comment already exists");
done = false;
}
// comment_data exists but parent comment hasn't been created yet
else if(comment_lookup.hasOwnProperty(parent_id)) {
//created_comments.push(create_comment(comments_data[parent_id]));
comment_section.appendChild(create_comment(comment_lookup[parent_id], false));
//console.log(i, " comment data exists");
//done = false;
}
// parent comment doesn't exists
else {
get(single_comment_url + parent_id, function(data){
var comment = create_comment(JSON.parse(data).data.children[0].data, false);
comment_section.appendChild();
add_parent_comments();
console.log(parent_id, " created via /api/info ???");
});
console.log(i, " doesn't exist");
}
}
if(!done) {
add_parent_comments();
} else {
fix_background_color();
sort_comments();
}
}
*/
/*
function get_comments(json) {
var comments = [];
var comments_json = json[1].data.children;
for(var i = 0; i < comments_json.length; i++) {
get_comment_data(comments_json[i], comments);
}
return comments;
}
var stripped_comment_tags = ["author_flair_css_class", "approved_by",
"approved_at_utc", "archived", "author_flair_text",
"body_html", "banned_at_utc", "banned_by",
"can_gild", "can_mod_post", "created", "collapsed", "collapsed_reason",
"downs", "distinguished", "gilded",
"edited", "likes", "link_id", "name", "mod_reports", "num_reports",
"removal_reason", "report_reasons",
"saved", "subreddit", "subreddit_type", "subreddit_name_prefixed", "subreddit_id",
"ups", "user_reports"];
// Strips unnecessary tags from comment, adds to comment array
// Do this recursivly to all replies
// Params comment:JSON object, comments: array
function get_comment_data(comment, comments) {
if(comment.kind !== "t1") {
alert(comment.kind);
return;
}
var comment_data = comment.data;
for(var i = 0; i < stripped_comment_tags.length; i++) {
delete comment_data[stripped_comment_tags[i]];
}
if(comment_data.replies) {
for(var i = 0; i < comment_data.replies.data.children.length; i++) {
get_comment_data(comment_data.replies.data.children[i], comments);
}
}
if(comment_data.parent_id.indexOf("_") !== -1) {
comment_data.parent_id = comment_data.parent_id.split("_")[1];
}
delete comment_data["replies"];
comments.push(comment_data);
}
*/
old_code:
"""headers = {'Authorization': 'bearer ' + token, 'User-Agent': USER_AGENT}
url = API_URL + THREAD_PATH.format(subreddit, thread_id)
response = requests.get(url, headers=headers)
thread_json = None
try:
thread_json = response.json()
except:
return 'Error parsing the thread'
thread = {
'body': get_thread_body(thread_json),
'comments': get_thread_comments(thread_json)
}
return str(get_thread_comments(thread_json))
#return str(get_thread_comments(thread_json)) + '/' + str(thread['body']['num_comments'])
def get_thread_body(thread_json):
body_json = thread_json[0]['data']['children'][0]['data']
filtered_body = {k:v for k,v in body_json.items() if k in FILTER_BODY}
return filtered_body
def get_thread_comments(thread_json):
comments_json = thread_json[1]['data']['children']
filtered_comments = []
for comment_json in comments_json:
comment_data = get_comment_data(comment_json)
filtered_comments.append(comment_data)
return filtered_comments
def get_comment_data(comment):
comment_data = comment['data']
filtered_comment = {k:v for k,v in comment_data.items() if k in FILTER_COMMENT}
filtered_comment['replies'] = []
if comment_data['replies']:
for child_comment in comment_data['replies']['data']['children']:
filtered_comment['replies'].append(get_comment_data(child_comment))
return filtered_comment
"""

9
secret.go Normal file
View file

@ -0,0 +1,9 @@
package main
const (
appName = ""
userName = ""
clientID = ""
clientSecret = ""
version = ""
)

180
server.go Normal file
View file

@ -0,0 +1,180 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"html/template"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"strings"
"time"
)
// consts
const (
userAgent = "Javascript:" + appName + ":" + version + "s (by /u/" + userName + ")"
tokenURL = "https://www.reddit.com/api/v1/access_token"
commentIDsURL = "https://api.pushshift.io/reddit/submission/comment_ids/"
commentsURL = "https://api.pushshift.io/reddit/comment/search?ids="
tmplFolder = "templates/"
errorHTML = "<h3 style=\"colorError;font-family: verdana, arial, helvetica, sans-serif;\">Error: %s</h3>"
)
var (
client = &http.Client{Timeout: time.Second * 10}
tokenData = url.Values{"grant_type": []string{"client_credentials"}}
templates = template.Must(template.ParseFiles(
tmplFolder+"header.html",
tmplFolder+"footer.html",
tmplFolder+"thread.html",
tmplFolder+"frontpage.html",
tmplFolder+"error.html",
))
accessLog *log.Logger
errorLog *log.Logger
)
type tokenResponse struct {
AccessToken string `json:"access_token"`
TokenType string `json:"token_type"`
ExpiresIn int `json:"expires_in"`
Scope string `json:"scope"`
}
type pushshiftAPI struct {
Data []string `json:"data"`
}
type pageData struct {
Token string
Subreddit string
ThreadID string
CommentIDs []string
}
func main() {
initLogging()
accessLog.Println("Staring server")
/*
s := &http.Server{
Addr: ":8080",
Handler: myHandler,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}*/
// Only serve local static files for debugging, otherwise use cdn.rawgit.com
fs := http.FileServer(http.Dir("static"))
http.Handle("/static/", http.StripPrefix("/static/", fs))
http.HandleFunc("/", pageHandler(mainHandler))
http.HandleFunc("/r/", pageHandler(threadHandler))
http.HandleFunc("/comments/", pageHandler(commentHandler))
http.ListenAndServe(":9000", nil)
}
func initLogging() {
accessFile, _ := os.OpenFile("access.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
errorFile, _ := os.OpenFile("error.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
accessLog = log.New(accessFile, "", log.Ldate|log.Ltime|log.Lshortfile)
errorLog = log.New(errorFile, "", log.Ldate|log.Ltime|log.Lshortfile)
}
// Wrap handle function for the access log
func pageHandler(fn func(http.ResponseWriter, *http.Request)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
accessLog.Println(r.URL.Path)
fn(w, r)
}
}
func renderTemplate(w http.ResponseWriter, pageName string, data interface{}) {
templates.ExecuteTemplate(w, "header.html", nil)
templates.ExecuteTemplate(w, pageName+".html", data)
templates.ExecuteTemplate(w, "footer.html", nil)
}
func mainHandler(w http.ResponseWriter, r *http.Request) {
renderTemplate(w, "frontpage", nil)
}
func threadHandler(w http.ResponseWriter, r *http.Request) {
pathParts := strings.Split(r.URL.Path, "/")
if len(pathParts) < 5 {
handleError(w, "Missing necessary parts of the URL")
return
}
resp, _ := client.Get(commentIDsURL + pathParts[4])
if resp.StatusCode != 200 {
handleError(w, "Trouble getting removed comments from pushshift")
return
}
token, err := getAPIToken()
if err != nil {
handleError(w, err.Error())
return
}
body, err := ioutil.ReadAll(resp.Body)
var dataStructure pushshiftAPI
json.Unmarshal(body, &dataStructure)
data := &pageData{
Token: token,
Subreddit: pathParts[2],
ThreadID: pathParts[4],
CommentIDs: dataStructure.Data,
}
renderTemplate(w, "thread", data)
}
func handleError(w http.ResponseWriter, msg string) {
renderTemplate(w, "error", msg)
errorLog.Println(msg)
}
func commentHandler(w http.ResponseWriter, r *http.Request) {
commentIDs := r.FormValue("c")
if commentIDs == "" {
handleError(w, "Not enough arguments provided")
return
}
resp, err := client.Get(commentsURL + commentIDs)
if err != nil {
handleError(w, "Trouble getting removed comments from pushshift")
return
}
body, _ := ioutil.ReadAll(resp.Body)
fmt.Fprintf(w, string(body))
}
func getAPIToken() (string, error) {
req, _ := http.NewRequest("POST", tokenURL, strings.NewReader(tokenData.Encode()))
req.Header.Add("User-Agent", userAgent)
req.SetBasicAuth(clientID, clientSecret)
resp, err := client.Do(req)
if err != nil {
return "", errors.New("Coulnd't get API token from Reddit")
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
var r tokenResponse
json.Unmarshal(body, &r)
return r.AccessToken, nil
}

BIN
static/favicon (copy).ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

View file

View file

@ -40,6 +40,10 @@ async function load_page() {
generate_comments(removed_comments)
}
// ------------------------------------------------------------------------------
// ----------------------- Functions for getting IDs ----------------------------
// ------------------------------------------------------------------------------
async function get_removed_comments() {
const ids_diff = all_ids.filter(x => !comment_ids.includes(x))
generate_comment_info(ids_diff.length)
@ -120,9 +124,9 @@ function extract_id_from_comment(comment) {
return replies_ids
}
// -------------------------------------------------------------------------------------
// ----------------- Comment generating functions ---------------------
// -------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------
// ----------------------- Comment generating functions -------------------------
// ------------------------------------------------------------------------------
async function generate_comments(removed_comments) {
const comment_section = document.createElement("div")
@ -199,10 +203,9 @@ function create_comments() {
}
}
}
// -------------------------------------------------------------------------------------
// --------------------- HTML-generating functions ----------------------
// -------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------
// ------------------------- HTML-generating functions --------------------------
// ------------------------------------------------------------------------------
function generate_thread(data) {
const thread = data[0].data.children[0].data
@ -260,9 +263,10 @@ function generate_comment_info(removed_comments) {
`
main_div.insertBefore(comment_info, loading_comments)
}
// -------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------
// ----------------------------- AJAX-functions ---------------------------------
// -------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------
async function fetch_multiple(url, data, json_walkdown=[], flattening=false) {
const responses = await Promise.all(split_array(data, max_ids_per_call).map(single_request_data => fetch(url + single_request_data.join())))
@ -272,10 +276,10 @@ async function fetch_multiple(url, data, json_walkdown=[], flattening=false) {
})
return flattening ? flatten_array(json) : json
}
// -------------------------------------------------------------------------------------
// ---------------- Other less interesting functions ---------------------
// -------------------------------------------------------------------------------------
// ------------------------------------------------------------------------------
// ---------------------- Other less interesting functions ----------------------
// ------------------------------------------------------------------------------
function split_array(array, size) {
const array_split = []

View file

@ -42,17 +42,17 @@ a:hover {
}
#header {
background-color: #181818;
background-color: #ca302c;/*#19171c;/*#181818*/
padding: 20px 10px 10px;
}
#header a {
color: #ddd;
color: #fff;/*#ddd;*/
transition: color 0.3s;
}
#header a:hover {
color: #ca302c;
color: #FF8B88;/*#ca302c*/;
text-decoration: none;
}
@ -66,6 +66,32 @@ a:hover {
margin: 15px;
}
#main-box {
margin: 0 auto;
padding: 20px;
border-radius: 5px;
color: #fff;
}
.error-box {
max-width: 500px;
background-color: #ca302c;
}
.frontpage-box {
max-width: 800px;
background-color: #161616;
}
.frontpage-box p {
margin: 15px;
}
.error-title {
margin: 0 0 15px 0;
}
#thread {
display: flex;
}

4
templates/error.html Normal file
View file

@ -0,0 +1,4 @@
<div id="main-box" class="error-box">
<h2 class="error-title">Error 500: Internal server error</h2>
<p><b>Details</b>: {{.}}</p>
</div>

View file

@ -1,6 +1,4 @@
<div id="debug"></div>
</div>
</div>
</body>
</html>

View file

@ -1,7 +1,41 @@
{% extends "main.html" %}
{% block body %}
frontpage
{% endblock %}
<div id="main-box" class="frontpage-box">
<h2 style="color:#CA302C; margin-top:5px">About</h2>
<p>
This is a site for viewing removed comments from Reddit.
It was created by Jesper Wrang and uses <a href="https://pushshift.io/">Jason Baumgartner</a> service for getting removed comments.</p>
<p>
<b>How to use</b>: go to any Reddit thread and replace the <i>reddit</i> in the URL with <i>removeddit</i>
<br>E.g.
<a href="/r/changelog/comments/6xfyfg/">
https://www.removeddit.com/r/changelog/comments/6xfyfg/
</a>
</p>
<p>
Note that this site is in it's early stages of development, meaning the site is unstable and will contain bugs.
If you find something that doesn't work please contact me at: <br><b>jesper (at) removeddit.com</b>
</p>
<h2 style="color:#239F2B">TODO</h2>
<ul>
<li>Collapsing comments</li>
<li>Getting removed selftext of thread</li>
</ul>
<h2 style="color:#1B767A">Acknowledgement</h2>
<p>
<a href="https://pushshift.io/">Jason Baumgartner</a> - for making all this possible with his amazing API
</p>
<p>
<a href="https://github.com/gamefreak">Scott McClaugherty</a> - for his reddit markdown parser
</p>
<p>
<a href="http://reddit.com/r/redditdev">/r/redditdev</a> - for helping with the Reddit API
</p>
<h2 style="color: #CA752C">Links/Contact</h2>
<p>For feedback and bug reports:
<ul>
<li>email: jesper (at) removeddit.com</li>
<li>reddit: <a href="https://www.reddit.com/user/Jubbeart/">/u/JubbeArt</a></li>
</ul>
<p>
<a href="https://github.com/JubbeArt/removeddit">Github for this project</a>
</p>
</div>

View file

@ -1,15 +1,14 @@
<!DOCTYPE html>
<html>
<head>
<title>Removeddit</title>
<meta charset="utf-8">
<link href="{{ url_for('static', filename='style.css') }}" rel="stylesheet">
<link rel="shotcut icon" href="{{ url_for('static', filename='favicon.ico') }}">
<title>Removeddit</title>
<meta charset="utf-8">
<link href="/static/style.css" rel="stylesheet">
<!--https://cdn.rawgit.com/JubbeArt/removeddit/5a25b161/static/style.css-->
<link rel="shotcut icon" href="/static/favicon.ico">
</head>
<body>
<header id="header">
<a href="https://jubbeart.com"><h1 id="header-title">Removeddit [beta]</h1></a>
</header>
<div id="main">
<header id="header">
<a href="/"><h1 id="header-title">Removeddit [beta]</h1></a>
</header>
<div id="main">

View file

@ -1,4 +0,0 @@
{% include "header.html" %}
{% block body%}
{% endblock %}
{% include "footer.html" %}

View file

@ -1,16 +1,9 @@
{% extends "main.html" %}
{% block body %}
<div id="loading-comments" class="loading-comments"></div>
<!-- https://rawgit.com/ -->
<script src="{{ url_for('static', filename='snuownd.js') }}"></script>
<script src="https://cdn.rawgit.com/showdownjs/showdown/1.7.2/dist/showdown.min.js"></script>
<script src="{{ url_for('static', filename='functions.js') }}"></script>
<script src="https://cdn.rawgit.com/JubbeArt/removeddit/5a25b161/static/snuownd.js"></script>
<script>
const thread_id = "{{ thread_id }}";
const token = "{{ token }}";
const subreddit = "{{ subreddit }}";
const all_ids = {{ comment_ids|safe }};
const thread_id = "{{ .ThreadID }}";
const token = "{{ .Token }}";
const subreddit = "{{ .Subreddit }}";
const all_ids = {{ .CommentIDs }};
</script>
<script src="{{ url_for('static', filename='script.js')}}"></script>
{% endblock %}
<script src="https://cdn.rawgit.com/JubbeArt/removeddit/5a25b161/static/script.js"></script>