From b74af1db3fbb27bd8245c7fa5f51cca93c93895f Mon Sep 17 00:00:00 2001 From: Jesper Wrang Date: Thu, 16 Nov 2017 18:41:32 +0100 Subject: [PATCH] Rewrote majority of the thread part --- compile.sh | 7 +- jst/comment.jst | 25 ++ jst/elasticCommentIDs.jst | 10 + jst/elasticComments.jst | 11 + jst/elasticSubreddit.jst | 30 ++ jst/elasticThread.jst | 11 + jst/thread.jst | 5 +- jst/threadInfo.jst | 4 + pug/base.pug | 1 - sass/post.sass | 140 +------ sass/thread.sass | 141 ++++++- search.html | 1 - static/css/style.css | 31 +- static/js/functions.js | 13 +- static/js/id.js | 8 - static/js/libraries/lodash.templates.min.js | 17 +- static/js/search.js | 11 - static/js/subreddit.js | 35 +- static/js/thread.js | 389 +++++--------------- subreddit.html | 1 - thread.html | 1 - 21 files changed, 373 insertions(+), 519 deletions(-) create mode 100644 jst/comment.jst create mode 100644 jst/elasticCommentIDs.jst create mode 100644 jst/elasticComments.jst create mode 100644 jst/elasticSubreddit.jst create mode 100644 jst/elasticThread.jst create mode 100644 jst/threadInfo.jst delete mode 100644 static/js/id.js diff --git a/compile.sh b/compile.sh index b24d2c5..4948d63 100755 --- a/compile.sh +++ b/compile.sh @@ -1,7 +1,12 @@ +### Use these for production + pug -P -o . pug/about.pug pug/thread.pug pug/subreddit.pug pug/search.pug sass --sourcemap=none --style expanded sass/main.sass static/css/style.css lodash include=defaultTo,toLower,includes,parseInt,has,isNil,uniq,property,map,forEach,join,sortBy,get,now,template -p -o static/js/libraries/lodash.custom.min.js lodash template="jst/*.jst" -p -o static/js/libraries/lodash.templates.min.js +### Use these in development + #pug --watch -P -o . pug/about.pug pug/thread.pug pug/subreddit.pug pug/search.pug -#sass --watch --sourcemap=none --style expanded sass/main.sass:static/css/style.css \ No newline at end of file +#sass --watch --sourcemap=none --style expanded sass/main.sass:static/css/style.css +#lodash template="jst/*.jst" -d -o static/js/libraries/lodash.templates.min.js diff --git a/jst/comment.jst b/jst/comment.jst new file mode 100644 index 0000000..1bf7d8a --- /dev/null +++ b/jst/comment.jst @@ -0,0 +1,25 @@ +<% +var isRemoved = _.has(comment,"removed"); +var isDeleted = _.has(comment,"deleted"); + +var commentCss = "comment comment-" + (isRemoved ? "removed" : (isDeleted ? "deleted" : (comment.depth % 2 == 0 ? "even" : "odd"))); +%> + +
+
+ [–] + + <%= comment.author+(isDeleted ? " (deleted by user)" : "") %> + + <%= Format.prettyScore(comment.score) %> point<%= ((comment.score == 1) ? "": "s") %> + <%= Format.prettyDate(comment.created_utc) %> +
+
+ <%= (comment.body === "[removed]" && isRemoved ? "

[likely removed by automoderator]

" : Format.parse(comment.body)) %> +
+
+ permalink + reddit +
+
+ diff --git a/jst/elasticCommentIDs.jst b/jst/elasticCommentIDs.jst new file mode 100644 index 0000000..2b8a06c --- /dev/null +++ b/jst/elasticCommentIDs.jst @@ -0,0 +1,10 @@ +{ + "query":{ + "ids":{ + "values":[<%= ids.join() %>] + } + }, + "_source":[ + "author","body","created_utc","parent_id","score" + ] +} \ No newline at end of file diff --git a/jst/elasticComments.jst b/jst/elasticComments.jst new file mode 100644 index 0000000..3ce0232 --- /dev/null +++ b/jst/elasticComments.jst @@ -0,0 +1,11 @@ +{ + "query":{ + "term":{ + "link_id":<%= id %> + } + }, + "_source":[ + "author","body","created_utc","parent_id","score" + ], + "size":50 +} \ No newline at end of file diff --git a/jst/elasticSubreddit.jst b/jst/elasticSubreddit.jst new file mode 100644 index 0000000..cc7bca8 --- /dev/null +++ b/jst/elasticSubreddit.jst @@ -0,0 +1,30 @@ +{ + "query":{ + "bool":{ + "must":[ + <% if(_.toLower(subreddit) !== "all") { %> + { + "term":{ + "subreddit":"<%= _.toLower(subreddit) %>" + } + }, + <% } %> + { + "range":{ + "created_utc":{ + "gt":<%= time %> + } + } + } + ] + } + }, + "_source":[ + "author","url","subreddit","link_flair_text","score","title","created_utc","num_comments","domain","permalink","id","thumbnail","thumbnail_height","thumbnail_width" + ], + "sort":[ + {"score":"desc"} + ], + "from":<%= (page-1)*postPerPage %>, + "size":<%= postPerPage %> +} \ No newline at end of file diff --git a/jst/elasticThread.jst b/jst/elasticThread.jst new file mode 100644 index 0000000..09e8b7a --- /dev/null +++ b/jst/elasticThread.jst @@ -0,0 +1,11 @@ +{ + "query":{ + "term":{ + "id":"<%= id %>" + } + }, + "_source":[ + "author","created_utc","domain","link_flair_text","num_comments","url","subreddit","score","title","selftext","permalink","thumbnail","thumbnail_height","thumbnail_width" + ], + "size":1 +} \ No newline at end of file diff --git a/jst/thread.jst b/jst/thread.jst index 6f48080..0e36bb8 100644 --- a/jst/thread.jst +++ b/jst/thread.jst @@ -1,5 +1,7 @@
+<% if(typeof postNr !== "undefined") { %> <%= postNr %> +<% } %>
<%= Format.prettyScore(thread.score) %>
@@ -17,10 +19,11 @@ <% } %>
+ + <%= thread.title %> <% if(!_.isNil(thread.link_flair_text)){ %> <%= thread.link_flair_text %> <% } %> - <%= thread.title %> (<%= thread.domain %>)
submitted <%= Format.prettyDate(thread.created_utc) %> by diff --git a/jst/threadInfo.jst b/jst/threadInfo.jst new file mode 100644 index 0000000..8ba0ec6 --- /dev/null +++ b/jst/threadInfo.jst @@ -0,0 +1,4 @@ +
+ removed comments: <%= removedComments %>/<%= totalComments %> (<%= (100 * removedComments / totalComments).toFixed(1) %>%) +
+
sorted by: top
\ No newline at end of file diff --git a/pug/base.pug b/pug/base.pug index 641a26b..e0c08c7 100644 --- a/pug/base.pug +++ b/pug/base.pug @@ -35,5 +35,4 @@ html script(src='/static/js/libraries/lodash.templates.min.js') script(src='/static/js/polyfills/promise.min.js') script(src='/static/js/polyfills/fetch.js') - script(src='/static/js/id.js') script(src='/static/js/functions.js') \ No newline at end of file diff --git a/sass/post.sass b/sass/post.sass index 44fca67..b205c04 100644 --- a/sass/post.sass +++ b/sass/post.sass @@ -1,131 +1,13 @@ -.thread - display: flex - flex-grow: 1 - margin-bottom: 7px - - .thumbnail - width: 70px - margin-right: 3px - - img - border-radius: 1px - - .thumbnail-default - background-image: url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png") - background-position: 0px -1099px - background-repeat: no-repeat - height: 50px - - .thumbnail-self - background-image: url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png") - background-position: 0px -1267px - background-repeat: no-repeat - height: 50px - - .thumbnail-image - background-image: url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png") - background-position: 0px -1043px - background-repeat: no-repeat - height: 50px - - .thumbnail-nsfw - background-image: url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png") - background-position: 0px -1155px - background-repeat: no-repeat - height: 50px - .thread-score-box - margin: 0 7px - width: 43px - - .vote - background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAcCAMAAADC4sagAAABFFBMVEUjJCQiIiIjIyMkIiIkJCQlJSUlJicmJiYnIyEnJycnKCsoKCgpLC8rIyErKyssMDUvLy8wJCAwMDAxMTEyMjI0NDQ2NjY3JR83Nzc5OTk7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NGRkZISEhKSkpKV2tLS0tMTExOTk5PT09QYHdSUlJTU1NVVVVWVlZXV1dXaINYWFhaWlpabYlbW1tecpBgYGBhYWFid5dkZGRle5xlfJ1pgKRrhKhshKlshapuLhZwirFyjbV0j7h4lL97mcZ9MBR/nsyCo9OOMxGcNQ+eNg+kNw6sOA2vOAy0OQy7Ogq/OwrBOwrDPAnLPQjPPQfTPgfbPwXjQQTrQgPzQwLKxGgxAAAAAXRSTlMAQObYZgAAAAlwSFlzAAALEwAACxMBAJqcGAAAAVlJREFUOMu9lF1TwjAQRTcMvmeQOi1QPmxFBQTxW1ERUUFEUVRU/P//w9yGmBTFyZPnpXNmbju76W6IJM8Uw49rLa6Tz4mpq/mCqY3Gvqnj6XQ61lp0mGd8favCwx2tow8wUlpwGWOZrNJKjXO+UVc6fJcMZzV7IswSOU/qej0l0rwyq33wphhAM1kWkcy70GB7mUfUNqG9V02PyPMTMs2Wig5RaTctwzxVXyPqvJh0VkJBIKJ4huncCRAvHOEZyGKegO4a6UDrIdLGCT4C6/QDsE7fA+v0HdAadWqk0SH9D3sgXkm4uJIDYKTFnwwXd3kMrNOnwDp9BmzS5QuTshN1iDRwfkxV9VJTJXLzSWYM+NzEEjWvFE2oGvCE7/2yDeL8riWzM8xmorRazLlNI2rdgNb3ZYLF1Es/t8VE7X6/39Yqlt4tLLwhiLq3XVNLfumP24fo3Opm+wLGCVa252Y8tQAAAABJRU5ErkJggg==) - background-repeat: no-repeat - margin-left: auto - margin-right: auto - width: 15px - height: 14px - cursor: pointer - - .upvote - background-position: -15px 0px - margin-top: 2px - - .downvote - background-position: -15px -14px - margin-top: 2px - - .thread-score - color: #646464 - font-size: 13px - font-weight: bold - text-align: center - - .thread-content - flex: 1 - float: left - margin-left: 3px - - .link-flair - color: #c8c8c8 - background-color: #404040 - margin-right: 5px - border: 1px solid #4d4d4d - border-radius: 2px - padding: 0 2px - font-size: 10px - - .thread-title - color: #a6a6a6 - margin-right: 5px - - .thread-title:hover - text-decoration: none - - .domain - color: #888 - font-size: 10px - - .thread-selftext - border: 1px solid #666 - border-radius: 7px - margin: 5px 0 7px - padding: 5px 10px - max-width: 840px - color: #ddd - font-size: 14px - - p - margin: 5px 0 - line-height: 20px - - a:hover - text-decoration: none - - p:first-child - margin-top: 0px +#comment-info + color: #ca302c + margin-top: 10px + font-weight: bold + padding-bottom: 5px + border-bottom: 1px dotted #808080 + font-size: 16px - p:last-child - margin-bottom: 0px - - - .thread-image - max-width: 768px - max-height: 768px - - .thread-info - color: #828282 - font-size: 10px - margin-top: 2px - - - .total-comments - color: #828282 - font-size: 10px - margin: 4px 0 - - - .grey-link, .grey-link:link, .grey-link:visited, .grey-link:hover, .grey-link:focus, .grey-link:active - color: #828282 - margin-right: 4px - \ No newline at end of file +#comment-sort + color: #808080 + font-size: 12px + margin: 5px 0 10px diff --git a/sass/thread.sass b/sass/thread.sass index b205c04..7c0d20a 100644 --- a/sass/thread.sass +++ b/sass/thread.sass @@ -1,13 +1,132 @@ -#comment-info - color: #ca302c - margin-top: 10px - font-weight: bold - padding-bottom: 5px - border-bottom: 1px dotted #808080 - font-size: 16px +.thread + display: flex + flex-grow: 1 + margin-bottom: 7px + + .thumbnail + width: 70px + margin-right: 3px + + img + border-radius: 1px + + .thumbnail-default + background-image: url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png") + background-position: 0px -1099px + background-repeat: no-repeat + height: 50px + + .thumbnail-self + background-image: url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png") + background-position: 0px -1267px + background-repeat: no-repeat + height: 50px + + .thumbnail-image + background-image: url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png") + background-position: 0px -1043px + background-repeat: no-repeat + height: 50px + + .thumbnail-nsfw + background-image: url("https://www.redditstatic.com/sprite-reddit.6Om8v6KMv28.png") + background-position: 0px -1155px + background-repeat: no-repeat + height: 50px + .thread-score-box + margin: 0 7px + width: 43px + + .vote + background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAcCAMAAADC4sagAAABFFBMVEUjJCQiIiIjIyMkIiIkJCQlJSUlJicmJiYnIyEnJycnKCsoKCgpLC8rIyErKyssMDUvLy8wJCAwMDAxMTEyMjI0NDQ2NjY3JR83Nzc5OTk7Ozs8PDw9PT0+Pj4/Pz9AQEBBQUFCQkJDQ0NGRkZISEhKSkpKV2tLS0tMTExOTk5PT09QYHdSUlJTU1NVVVVWVlZXV1dXaINYWFhaWlpabYlbW1tecpBgYGBhYWFid5dkZGRle5xlfJ1pgKRrhKhshKlshapuLhZwirFyjbV0j7h4lL97mcZ9MBR/nsyCo9OOMxGcNQ+eNg+kNw6sOA2vOAy0OQy7Ogq/OwrBOwrDPAnLPQjPPQfTPgfbPwXjQQTrQgPzQwLKxGgxAAAAAXRSTlMAQObYZgAAAAlwSFlzAAALEwAACxMBAJqcGAAAAVlJREFUOMu9lF1TwjAQRTcMvmeQOi1QPmxFBQTxW1ERUUFEUVRU/P//w9yGmBTFyZPnpXNmbju76W6IJM8Uw49rLa6Tz4mpq/mCqY3Gvqnj6XQ61lp0mGd8favCwx2tow8wUlpwGWOZrNJKjXO+UVc6fJcMZzV7IswSOU/qej0l0rwyq33wphhAM1kWkcy70GB7mUfUNqG9V02PyPMTMs2Wig5RaTctwzxVXyPqvJh0VkJBIKJ4huncCRAvHOEZyGKegO4a6UDrIdLGCT4C6/QDsE7fA+v0HdAadWqk0SH9D3sgXkm4uJIDYKTFnwwXd3kMrNOnwDp9BmzS5QuTshN1iDRwfkxV9VJTJXLzSWYM+NzEEjWvFE2oGvCE7/2yDeL8riWzM8xmorRazLlNI2rdgNb3ZYLF1Es/t8VE7X6/39Yqlt4tLLwhiLq3XVNLfumP24fo3Opm+wLGCVa252Y8tQAAAABJRU5ErkJggg==) + background-repeat: no-repeat + margin-left: auto + margin-right: auto + width: 15px + height: 14px + cursor: pointer + + .upvote + background-position: -15px 0px + margin-top: 2px + + .downvote + background-position: -15px -14px + margin-top: 2px + + .thread-score + color: #646464 + font-size: 13px + font-weight: bold + text-align: center + + .thread-content + flex: 1 + float: left + margin-left: 3px + + .link-flair + display: inline-block + color: #c8c8c8 + background-color: #404040 + margin-right: 5px + border: 1px solid #4d4d4d + border-radius: 2px + padding: 0 2px + font-size: 10px + + .thread-title + color: #a6a6a6 + margin-right: 5px + + .thread-title:hover + text-decoration: none + + .domain + color: #888 + font-size: 10px + + .thread-selftext + border: 1px solid #666 + border-radius: 7px + margin: 5px 0 7px + padding: 5px 10px + max-width: 840px + color: #ddd + font-size: 14px + + p + margin: 5px 0 + line-height: 20px + + a:hover + text-decoration: none + + p:first-child + margin-top: 0px -#comment-sort - color: #808080 - font-size: 12px - margin: 5px 0 10px + p:last-child + margin-bottom: 0px + + + .thread-image + max-width: 768px + max-height: 768px + + .thread-info + color: #828282 + font-size: 10px + margin-top: 2px + + + .total-comments + color: #828282 + font-size: 10px + margin: 4px 0 + + + .grey-link, .grey-link:link, .grey-link:visited, .grey-link:hover, .grey-link:focus, .grey-link:active + color: #828282 + margin-right: 4px + \ No newline at end of file diff --git a/search.html b/search.html index b0859b3..23829a1 100644 --- a/search.html +++ b/search.html @@ -23,7 +23,6 @@ - diff --git a/static/css/style.css b/static/css/style.css index 0994609..88b6cba 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -165,6 +165,21 @@ header #status #status-image { background-color: #161616; } +#comment-info { + color: #ca302c; + margin-top: 10px; + font-weight: bold; + padding-bottom: 5px; + border-bottom: 1px dotted #808080; + font-size: 16px; +} + +#comment-sort { + color: #808080; + font-size: 12px; + margin: 5px 0 10px; +} + .thread { display: flex; flex-grow: 1; @@ -234,6 +249,7 @@ header #status #status-image { margin-left: 3px; } .thread .link-flair { + display: inline-block; color: #c8c8c8; background-color: #404040; margin-right: 5px; @@ -294,21 +310,6 @@ header #status #status-image { margin-right: 4px; } -#comment-info { - color: #ca302c; - margin-top: 10px; - font-weight: bold; - padding-bottom: 5px; - border-bottom: 1px dotted #808080; - font-size: 16px; -} - -#comment-sort { - color: #808080; - font-size: 12px; - margin: 5px 0 10px; -} - .post-rank { min-width: 20px; color: #505050; diff --git a/static/js/functions.js b/static/js/functions.js index 3d5207a..ad86b40 100644 --- a/static/js/functions.js +++ b/static/js/functions.js @@ -89,14 +89,13 @@ var Fetch2 = (function(){ var ElasticSearch = (function() { var elastic = "https://elastic.pushshift.io"; - //var elasticThreads: elastic+"/rs/submissions/_search?source=" - -return { - subreddit: function(){}, - thread: function(){}, - comment: function(){} -}; + return { + thread: function(){}, + comment: function(){}, + threadURL: elastic+"/rs/submissions/_search?source=", + commentURL: elastic+"/rc//comments/_search?source=" + }; })(); // HTML parsing diff --git a/static/js/id.js b/static/js/id.js deleted file mode 100644 index d7f33b0..0000000 --- a/static/js/id.js +++ /dev/null @@ -1,8 +0,0 @@ -// Change this to your own client ID: https://www.reddit.com/prefs/apps -// Guide about it here: https://github.com/JubbeArt/removeddit#the-i-just-want-to-get-this-shit-running-guide -// NOTE: The app NEEDS TO BE an installed app and NOT a web apps -var clientID = ""; - -if(clientID === "") { - alert("Missing client-ID for Reddit API. See the file \"static/js/id.js\""); -} \ No newline at end of file diff --git a/static/js/libraries/lodash.templates.min.js b/static/js/libraries/lodash.templates.min.js index 46f7f14..0be859f 100644 --- a/static/js/libraries/lodash.templates.min.js +++ b/static/js/libraries/lodash.templates.min.js @@ -3,10 +3,15 @@ * Lodash (Custom Build) lodash.com/license | Underscore.js underscorejs.org/LICENSE * Build: `lodash template="jst/*.jst" -p -o static/js/libraries/lodash.templates.min.js` */ -;(function(){var e=typeof self=="object"&&self&&self.Object===Object&&self,e=typeof global=="object"&&global&&global.Object===Object&&global||e||Function("return this")(),f=typeof exports=="object"&&exports&&!exports.nodeType&&exports,g=f&&typeof module=="object"&&module&&!module.nodeType&&module,c=e._||{},d={subredditInfo:function(a){a||(a={});var b;with(a)a=""+('
\n\ttop post of /r/'+(null==(b=subreddit)?"":b)+' from:\n\t\t\t\t\t\n
"},subredditPagination:function(a){a||(a={});var b;with(a){a='"},thread:function(a){a||(a={});var b;with(a){a=""+('
\n\t'+(null==(b=postNr)?"":b)+'\n\t
\n\t\t
\n\t\t
'+(null==(b=Format.prettyScore(thread.score))?"":b)+'
\n\t\t
\n\t
\n');var d=c.replace(thread.url,"https://www.reddit.com","https://www.removeddit.com");if(c.includes(["self","default","image","nsfw"],thread.thumbnail))a+='\n\t\t\n';else{ -var e=.5*c.defaultTo(thread.thumbnail_width,140),f=.5*c.defaultTo(thread.thumbnail_height,140);a+='\t\n\t\t\t\n\t\t\n'}a+='\n\t
\n',c.isNil(thread.link_flair_text)||(a+='\n\t\t'+(null==(b=thread.link_flair_text)?"":b)+"\n"),a+='\t\n\t\t'+(null==(b=thread.title)?"":b)+'\n\t\t('+(null==(b=thread.domain)?"":b)+')\n\t\t
\n\t\t\tsubmitted '+(null==(b=Format.prettyDate(thread.created_utc))?"":b)+' by\n\t\t\t'+(null==(b=thread.author)?"":b)+"\n\t\t", +;(function(){var e=typeof self=="object"&&self&&self.Object===Object&&self,e=typeof global=="object"&&global&&global.Object===Object&&global||e||Function("return this")(),f=typeof exports=="object"&&exports&&!exports.nodeType&&exports,h=f&&typeof module=="object"&&module&&!module.nodeType&&module,c=e._||{},d={comment:function(a){a||(a={});var b,g="";with(a){a=c.has(comment,"removed");var d=c.has(comment,"deleted"),e="comment comment-"+(a?"removed":d?"deleted":0==comment.depth%2?"even":"odd"),g=g+('\n\n
\n\t
\n\t\t[–]\n\t\t\n\t\t\t'+(null==(b=comment.author+(d?" (deleted by user)":""))?"":b)+'\n\t\t\n\t\t'+(null==(b=Format.prettyScore(comment.score))?"":b)+" point"+(null==(b=1==comment.score?"":"s")?"":b)+'\n\t\t'+(null==(b=Format.prettyDate(comment.created_utc))?"":b)+'\n\t
\n\t
\n\t\t'+(null==(b="[removed]"===comment.body&&a?"

[likely removed by automoderator]

":Format.parse(comment.body))?"":b)+'\n\t
\n\t
\n\t\tpermalink\n\t\treddit\n\t
\n
\n\n'); +}return g},elasticCommentIDs:function(a){a||(a={});var b;with(a)a=""+('{\n\t"query":{\n\t\t"ids":{\n\t\t\t"values":['+(null==(b=ids.join())?"":b)+']\n\t\t}\n\t},\n\t"_source":[\n\t\t"author","body","created_utc","parent_id","score"\n\t]\n}');return a},elasticComments:function(a){a||(a={});var b;with(a)a=""+('{\n\t"query":{\n\t\t"term":{\n\t\t\t"link_id":'+(null==(b=id)?"":b)+'\n\t\t}\n\t},\n\t"_source":[\n\t\t"author","body","created_utc","parent_id","score"\n\t],\n\t"size":50\n}');return a},elasticSubreddit:function(a){ +a||(a={});var b;with(a)a='{\n\t"query":{\n\t\t"bool":{\n\t\t\t"must":[\n\t\t\t\t',"all"!==c.toLower(subreddit)&&(a+='\n\t\t\t\t{\n\t\t\t\t\t"term":{\n\t\t\t\t\t\t"subreddit":"'+(null==(b=c.toLower(subreddit))?"":b)+'"\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t\t'),a+='\n\t\t\t\t{\n\t\t\t\t\t"range":{\n\t\t\t\t\t\t"created_utc":{\n\t\t\t\t\t\t\t"gt":'+(null==(b=time)?"":b)+'\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t},\n\t"_source":[\n\t\t"author","url","subreddit","link_flair_text","score","title","created_utc","num_comments","domain","permalink","id","thumbnail","thumbnail_height","thumbnail_width"\n\t],\n\t"sort":[\n\t\t{"score":"desc"}\n\t],\n\t"from":'+(null==(b=(page-1)*postPerPage)?"":b)+',\n\t"size":'+(null==(b=postPerPage)?"":b)+"\n}"; +return a},elasticThread:function(a){a||(a={});var b;with(a)a=""+('{\n\t"query":{\n\t\t"term":{\n\t\t\t"id":"'+(null==(b=id)?"":b)+'"\n\t\t}\n\t},\n\t"_source":[\n\t\t"author","created_utc","domain","link_flair_text","num_comments","url","subreddit","score","title","selftext","permalink","thumbnail","thumbnail_height","thumbnail_width"\n\t],\n\t"size":1\n}');return a},search:function(a){with(a||(a={}),a);return'\n\tvar radioInput = function(name, displayNames, values) {\n\t\tvar inputs = "";\n\t\t\n\t\tfor(var i = 0, len = displayNames.length; i < len; i++){\n\t\t\tinputs += \'\';\n\t\t\tinputs += \'\';\n\t\t\tinputs += \'\';\n\t\t}\n\t\treturn inputs;\n\t};\n\n\tvar textInput = function(name){\n\t\treturn \'\'; \n\t};\n\n\tvar label = function(name, display) {\n\t\treturn \'\';\n\t};\n\n\tvar inputRow = function(text, input) {\n\t\treturn \'
\'+text+\'\'+input+"
";\n\t};\n\n\tvar selectTime = function() {\n\t\tvar values = ["hour", "12hour", "day", "week", "month", "6month", "year", "all"];\n\t\tvar display = ["past hour", "past 12 hours", "past day", "past week", "past month", "past 6 months", "past year", "all time"];\n\t\tvar html = \'";\n\t};\n\n\tvar select = function(name, display, values){\n\t\tvar html = \'";\n\t};\n\nreturn {\n\tcreateSearch: function(){\n\t\tvar searchBox = document.createElement("div");\n\t\tsearchBox.id = "main-box";\n\t\tsearchBox.className = "search-box";\n\t/*\n\t\tComments:\n\t\ttext\t\t\t(body)\t\tstring\n\t\ttitle \t\t(title)\t\tstring\n\t\tsubreddit\t\t(subreddit)\tstring\n\t\tauthor\t\t(author)\t\tstring\n\t\tover_18\t\t(over_18)\t\tsfw,nsfw,both *\n\t\tlocked\t\t(locked)\t\tbool *\n\t\tbetween\t\t(after/before)\t"date"\n\t\tsort \t\t(sort)\t\tasc,desc\n\t*/\n\t\tvar html = inputRow(\'Im looking for: \', radioInput("thread", ["Thread","Comment"], ["true", "false"]));\t\t\n\t\thtml += inputRow(label("text", "Text"), textInput("text"));\n\t\thtml += inputRow(label("title","Title"), textInput("title"));\n\t\thtml += inputRow(label("subreddit", "Subreddit"), textInput("subreddit"));\n\t\thtml += inputRow(label("author", "Author"), textInput("author"));\n\t\thtml += inputRow("Over 18: ", radioInput("over_18", ["Both","NSFW","SFW"], ["both","nsfw","sfw"]));\n\t\thtml += inputRow("Locked: ", radioInput("locked", ["Both","True", "False"], ["both","true","false"]));\n\t\thtml += inputRow("Removed: ", select("removed", ["Removed and non-removed", "Only removed", "Only deleted"], ["all", "removed", "deleted"]));\n\t\thtml += inputRow("From:", selectTime());\n\t\thtml += inputRow("Sort: ", select("sort", ["Highest score","Lowest score","Newest","Oldest"], ["score_desc","score_asc","time_desc","time_asc"]));\n\t\thtml += \'\';\n\t\thtml += \'\';\n\t\tsearchBox.innerHTML = html;\n\t\n\t\tmainDiv.appendChild(searchBox);'; +},subredditInfo:function(a){a||(a={});var b;with(a)a=""+('
\n\ttop post of /r/'+(null==(b=subreddit)?"":b)+' from:\n\t\t\t\t\t\n
"},subredditPagination:function(a){a||(a={});var b;with(a){a='"},thread:function(a){a||(a={});var b;with(a){a='
\n',typeof postNr!="undefined"&&(a+='\n\t'+(null==(b=postNr)?"":b)+"\n"),a+='\n\t
\n\t\t
\n\t\t
'+(null==(b=Format.prettyScore(thread.score))?"":b)+'
\n\t\t
\n\t
\n'; +var d=c.replace(thread.url,"https://www.reddit.com","https://www.removeddit.com");if(c.includes(["self","default","image","nsfw"],thread.thumbnail))a+='\n\t\t\n';else{var e=.5*c.defaultTo(thread.thumbnail_width,140),f=.5*c.defaultTo(thread.thumbnail_height,140);a+='\t\n\t\t\t\n\t\t\n'; +}a+='\n\t
\n\n\t\t'+(null==(b=thread.title)?"":b)+"\n",c.isNil(thread.link_flair_text)||(a+='\n\t\t'+(null==(b=thread.link_flair_text)?"":b)+"\n"),a+='\t\n\t\t('+(null==(b=thread.domain)?"":b)+')\n\t\t
\n\t\t\tsubmitted '+(null==(b=Format.prettyDate(thread.created_utc))?"":b)+' by\n\t\t\t'+(null==(b=thread.author)?"":b)+"\n\t\t", Reddit.isAll&&(a+='\n\t\t\tto /r/'+(null==(b=thread.subreddit)?"":b)+"\n\t\t"),a+='\t\t\n\t\t
\n\t\t
\n\t\t\t\n\t\t\t\t'+(null==(b=thread.num_comments)?"":b)+' comments\n\t\t\t\n\t\t\t\n\t\t\t\treddit\n\t\t\t\n\t\t
\n\t
\n
'; -}return a}};typeof define=="function"&&typeof define.amd=="object"&&define.amd? define(["lodash"],function(t){c=t,t.templates=t.extend(t.templates||{},d)}):g?(c=require("lodash"),(g.exports=d).templates=d,f.templates=d):c&&(c.templates=c.extend(c.templates||{},d))}).call(this); \ No newline at end of file +}return a},threadInfo:function(a){a||(a={});var b;with(a)a=""+('
\n\tremoved comments: '+(null==(b=removedComments)?"":b)+"/"+(null==(b=totalComments)?"":b)+" ("+(null==(b=(100*removedComments/totalComments).toFixed(1))?"":b)+'%)\n
\n
sorted by: top
');return a}};typeof define=="function"&&typeof define.amd=="object"&&define.amd? define(["lodash"],function(t){c=t,t.templates=t.extend(t.templates||{},d)}):h?(c=require("lodash"),(h.exports=d).templates=d, +f.templates=d):c&&(c.templates=c.extend(c.templates||{},d))}).call(this); \ No newline at end of file diff --git a/static/js/search.js b/static/js/search.js index 27a85be..0c3575f 100644 --- a/static/js/search.js +++ b/static/js/search.js @@ -25,17 +25,6 @@ return { } }; })(); -/* -Comments: -text (body) string -title (title) string -subreddit (subreddit) string -author (author) string -over_18 (over_18) sfw,nsfw,both * -locked (locked) bool * -between (after/before) "date" -sort (sort) asc,desc -*/ var Vars = (function(){ var lookup = {}; diff --git a/static/js/subreddit.js b/static/js/subreddit.js index 1410cbf..f00d428 100644 --- a/static/js/subreddit.js +++ b/static/js/subreddit.js @@ -4,10 +4,17 @@ var app = (function(){ return { loadPage: function() { Status.loading("Loading subreddit..."); - document.title = Reddit.isAll ? 'all subreddits' : Reddit.subreddit; + document.title = Reddit.isAll ? "all subreddits" : Reddit.subreddit; HTML.main.innerHTML += _.templates.subredditInfo({subreddit: Reddit.subreddit, time: Vars.time}); - Fetch2.get(ElasticSearch.subreddit(Reddit.subreddit, Time.difference(Vars.time), Vars.page), "Could not get removed posts") + var urlData = { + subreddit: Reddit.subreddit, + time: Time.difference(Vars.time), + page: Vars.page, + postPerPage: Vars.postPerPage + }; + + Fetch2.get(ElasticSearch.threadURL+_.templates.elasticSubreddit(urlData), "Could not get removed posts") .then(Fetch2.json) .then(function(json){ var startPostNr = (Vars.page - 1) * Vars.postPerPage + 1; @@ -38,30 +45,6 @@ var Vars = (function(){ }; })(); -var ElasticSearch = (function(){ -return { - subreddit: function(subreddit, time, page){ - var condition = '[{"term":{"subreddit":"'+_.toLower(subreddit)+'"}},'+ - '{"range":{"created_utc":{"gt":'+time+'}}}]'; - - if(Reddit.isAll) { - condition = '{"range":{"created_utc":{"gt":'+time+'}}}'; - } - - return 'https://elastic.pushshift.io/rs/submissions/_search?source={'+ - '"query":{'+ - '"bool":{'+ - '"must":'+condition+ - '}'+ - '},'+ - '"_source":["author","url","subreddit","link_flair_text","score","title","created_utc","selftext","num_comments","domain","permalink","id","thumbnail","thumbnail_height","thumbnail_width"],'+ - '"sort":[{"score":"desc"}],'+ - '"from":'+(page-1)*Vars.postPerPage+','+ - '"size":'+Vars.postPerPage+'}'; - } -}; -})(); - if(isSupported) { app.loadPage(); } \ No newline at end of file diff --git a/static/js/thread.js b/static/js/thread.js index 4ddddd1..cf8d7b7 100644 --- a/static/js/thread.js +++ b/static/js/thread.js @@ -4,282 +4,121 @@ var app = (function(){ return { loadPage: function() { // Just fucking crach if the user is retarded - if(_.isUndefined(Reddit.threadID) || _.isUndefined(Reddit.subreddit)) { + if(_.isNil(Reddit.threadID)) { Status.error("Missing necessary parts of the URL"); return; } Status.loading("Loading thread..."); - Reddit.fetchToken() - .then(function(){ - return Promise.all([ - fetch(URLs.thread, Reddit.init) - .then(Fetch2.json) - .then(ThreadHTML.createThread) - .catch(function(error) { - return Promise.reject("Could not get thread from Reddit"); - }), + var urlData = {id: parseInt(Reddit.threadID, 36)}; - fetch(URLs.pushshiftIDs) - .then(Fetch2.json) - .then(_.property("data")) - .catch(function(error) { - return Promise.reject("Could not get removed comment IDs"); - }), - ]) - }) - .then(function(results) { - var thread = results[0]; - Comments.allIDs = results[1]; - - Status.loading("Loading comments from reddit..."); - HandleIDs.normal(thread); - return HandleIDs.morechildren() - .catch(function(error){ - return Promise.reject("Could not get comments from Reddit (moreChildren)"); - }); - }) - .then(function(){ - return Promise.all(_.map(_.uniq(Comments.countinuethread), function(id) { - return fetch(URLs.thread+"/_/"+id.split("_")[1], Reddit.init) - .then(Fetch2.json) - .catch(function(error){ return Promise.reject("Could not get comments from Reddit (continueThisThread)") }) - })) - .catch(function(error){ - return Promise.reject("Could not get comments from Reddit (continueThisThread)"); - }); - }) - .then(function(smallerThreads){ - _.forEach(smallerThreads, function(thread){ - HandleIDs.normal(thread); + Promise.all([ + Fetch2.get(ElasticSearch.threadURL+_.templates.elasticThread(urlData), "Could not get thread") + .then(Fetch2.json) + .then(function(json){ + HTML.main.innerHTML += _.templates.thread({thread: json.hits.hits[0]._source}); + return json; }) - Status.loading("Getting removed comments..."); - HandleIDs.removed(); - ThreadHTML.createCommentInfo(Comments.removed.length); - return Fetch2.multiple(URLs.format(URLs.pushshiftComments, Comments.removed), null, "data") - .catch(function(error){ - return Promise.reject("Could not get removed comments"); - }); + .catch(Fetch2.handleError), + + Fetch2.get(ElasticSearch.commentURL+_.templates.elasticComments(urlData), "Could not get removed comments") + .then(Fetch2.json) + .catch(Fetch2.handleError) + ]) + .then(function(promises){ + var commentData = { + removedComments: promises[1].hits.total, + totalComments: promises[0].hits.hits[0]._source.num_comments + }; + + HTML.main.innerHTML += _.templates.threadInfo(commentData); + Comments.addComments(promises[1]); + Status.loading("Getting remaining comments..."); }) - .then(function(removedComments){ - Status.loading("Generating comments..."); - return Comments.generate(removedComments); + .then(function getRemainingComments() { + return Fetch2.get(ElasticSearch.commentURL+_.templates.elasticCommentIDs({ids:Comments.missing}), "Could not get remaining comments") + .then(Fetch2.json) + .then(function(json){ + Comments.addComments(json); + + if(Comments.missing.length !== 0) { + getRemainingComments(); + } + }); + }) + .then(function(){ + // Make sure to get the permalink comment (even if it's not related to removed/deleted stuff) + if(Comments.isPermalink && !_.includes(Comments.IDs, Comments.getRoot())) { + var urlData = {ids: [Comments.to10(Comments.getRoot())]}; + return Fetch2.get(ElasticSearch.commentURL+_.templates.elasticCommentIDs(urlData), "Could not find the specific comment asked for") + .then(Fetch2.json) + .then(function(json){ + console.log(json.hits.hits) + Comments.addComments(json); + }); + } }) .then(function(){ + HTML.main.innerHTML += '
'; + Status.loading("Generating comments..."); + _.forEach(Comments.IDs, function(id){ + HTML.main.innerHTML += _.templates.comment({id:id, comment:Comments.lookup[id]}); + }); + // generate comments Status.success(); }) - .catch(function(error) { - if(_.includes(_.toLower(error), "error")) { - Status.error(error); - } else { - Status.error("Error: "+error); - } - }); + .catch(Fetch2.handleError); } }})(); -// ------------------------------------------------------------------------------ -// ----------------------- Store and genrates comments -------------------------- -// ------------------------------------------------------------------------------ + var Comments = (function() { - var totalComments; - var lookup = {}; - var toBeCreated = []; - - var getParentComments = function(toLookup){ - var newCommentsToLookup = []; - var commentsToFetch = []; - - _.forEach(toLookup, function(id){ - var parentID = lookup[id].parent_id.split("_")[1]; - - if(parentID === Reddit.threadID) {} // Has no parent (is parent of thread) - else if(_.includes(Comments.toBeCreated, parentID)) {} // Parent already exists, do nothing - else if(_.includes(newCommentsToLookup, parentID)) {} // Parent already exists (this iteration) - else if(_.has(lookup, parentID)) { - newCommentsToLookup.push(parentID); - } else{ - commentsToFetch.push(parentID); - } - - Comments.toBeCreated.push(id); - }); - - return new Promise(function(resolve, reject){ - if(_.uniq(commentsToFetch).length !== 0) { - fetch(URLs.singleComments + _.join(_.map(_.uniq(commentsToFetch),function(comments){ - return "t1_" + comments; - })), Reddit.init) - .then(Fetch2.json) - .then(function(json){ - _.forEach(json.data.children, function(comment) { - lookup[comment.data.id] = comment.data; - newCommentsToLookup.push(comment.data.id); - }); - }) - .then(function(){ - resolve(); - }); - - } else { - resolve(); - } - }) - .then(function(){ - if(newCommentsToLookup.length !== 0) { - return getParentComments(newCommentsToLookup) - } - }); - - }; - return { - ids: [], // The comments we found - morechildren: [], - countinuethread: [], - - allIDs: [], // All the comments that we were suppose to find - removed: [], - deleted: [], - toBeCreated: toBeCreated, - lookup: lookup, + IDs: [], // The comments we found + lookup: {}, + missing: [], - getTotalComments: function() { return totalComments; }, - setTotalComments: function(total) { totalComments = total; }, - - getRoot: function() { - if(Reddit.permalink !== undefined && Reddit.permalink === "") { - return Reddit.threadID; - } - - if(Reddit.permalink === undefined) { - return Reddit.threadID; - } - - if(_.has(Comments.lookup, Reddit.permalink)) { - return Comments.lookup[Reddit.permalink].parent_id.split("_")[1]; - } - return ""; + to36: function(id){ + return parseInt(id).toString(36); }, - generate: function(removedComments) { - removedComments.forEach(function(comment){ - if(_.includes(Comments.deleted, comment.id)) { - comment["deleted"] = true; - } else { - comment["removed"] = true; - } - - Comments.lookup[comment.id] = comment; + + to10: function(id){ + return parseInt(id, 36); + }, + + addComments: function(json){ + Comments.IDs = _.union(Comments.IDs, _.map(json.hits.hits, function(comment){ + return Comments.to36(comment._id); + })); + + _.forEach(json.hits.hits, function(comment){ + comment._source.parent_id = _.isNil(comment._source.parent_id) ? Reddit.threadID : Comments.to36(comment._source.parent_id); + Comments.lookup[Comments.to36(comment._id)] = comment._source; }); + + Comments.missing = _.reduce(json.hits.hits, function(array, comment){ + if(comment._source.parent_id !== Reddit.threadID && !_.includes(Comments.IDs, comment._source.parent_id)) { + array.push(Comments.to10(comment._source.parent_id)); + } + return array; + }, []); + }, - Comments.removed = _.map(removedComments, function(comment){ - return comment.id; - }); - - return getParentComments(Comments.removed) - .then(function(){ - ThreadHTML.createCommentSection(); - ThreadHTML.createComments(); - }) - } -}})(); - - - -// ------------------------------------------------------------------------------ -// ----------------- Handle comments from different requests -------------------- -// ------------------------------------------------------------------------------ -var HandleIDs = (function(){ - var normal = function(thread){ - return _.flatten(_.map(thread[1].data.children, Extract.normal)) - }; - - var morechildren = function(){ - return Promise.all(_.map(_.uniq(Comments.morechildren), function(idArray){ - return Fetch2.multiple(URLs.format(URLs.moreChildren, idArray), Reddit.init); - })) - .then(function(responseArrays){ - Comments.morechildren.length = 0; - _.forEach(responseArrays, function(responseArray){ - _.forEach(responseArray, function(response){ - _.forEach(response.jquery[10][3][0], function(comment){ - Extract.normal(comment); - }) - }); - }); - }).then(function(){ - if(Comments.morechildren.length !== 0) { - return morechildren(); - } - }); - }; - - var removed = function(){ - Comments.removed = _.difference(Comments.allIDs, Comments.ids); - - Comments.ids.forEach(function(id){ - if(! _.has(Comments.lookup, id)) { - return; - } - - if(Comments.lookup[id].body === "[removed]") { - Comments.removed.push(id); - } else if (Comments.lookup[id].body === "[deleted]"){ - Comments.removed.push(id); - Comments.deleted.push(id); - } - }); - - Comments.removed = _.uniq(Comments.removed); - }; - - return { - normal: normal, - morechildren: morechildren, - removed: removed - }; -})(); - - -// ------------------------------------------------------------------------------ -// ----------------------- Extract ID from comments ----------------------------- -// ------------------------------------------------------------------------------ -var Extract = (function(){ - var normal = function(comment){ - var data = comment.data; - - if(comment.kind == "more") { // "Show more"-comment - if(data.id === "_") { // = "continue this thread" comment - Comments.countinuethread.push(data.parent_id); - } else if(data.children.length < data.count){ // "Load more"-comment (that is missing some of its children) - Comments.morechildren.push(data.children); - } - Comments.ids.push.apply(Comments.ids, data.children); - } else { // Normal comment - if(data.replies) { - data.replies.data.children.forEach(function(child){ - normal(child); - }); - delete data.replies; - } - - Comments.ids.push(data.id); - Comments.lookup[data.id] = data; + getRoot: function() { + if(Reddit.permalink !== undefined && Reddit.permalink !== "") { + return Reddit.permalink; } - }; - return { - normal: normal - }; + return Reddit.threadID; + }, + + isPermalink: (Reddit.permalink !== undefined && Reddit.permalink !== "") +}; })(); - -// ------------------------------------------------------------------------------ -// ---------------------------- Generating HTML --------------------------------- -// ------------------------------------------------------------------------------ +/* var ThreadHTML = (function(){ - var mainDiv = document.getElementById("main"); // I actually haven't found a better way of doing this... // Imgur has image-links with no indication that they are actually images var imageHosts = ["i.redd.it", "flickr.com", "i.imgur.com", "imgur.com", "m.imgur.com"]; @@ -330,24 +169,6 @@ var ThreadHTML = (function(){ return data; }; - var createCommentInfo = function(totalRemovedComments){ - var div = document.createElement("div"); - div.innerHTML = ' \ -
\ - removed comments: '+totalRemovedComments+'/'+Comments.getTotalComments()+' ('+(100 * totalRemovedComments / Comments.getTotalComments()).toFixed(1)+'%) \ -
\ -
sorted by: top
\ - '; - - mainDiv.appendChild(div); - }; - - var createCommentSection = function(){ - var div = document.createElement("div"); - div.id = Comments.getRoot(); - mainDiv.appendChild(div); - }; - var createComments = function(){ var commentsToCreate = _.sortBy(_.uniq(Comments.toBeCreated), function(id) { return Comments.lookup[id].score; @@ -379,39 +200,7 @@ var ThreadHTML = (function(){ } }; - var createComment = function(comment){ - var isRemoved = _.has(comment,"removed"); - var isDeleted = _.has(comment,"deleted"); - - // Sorry - var commentCss = "comment-" + (isRemoved ? "removed" : (isDeleted ? "deleted" : (comment.depth % 2 == 0 ? "even" : "odd"))); - - var commentDiv = document.createElement("div"); - commentDiv.id = comment.id; - commentDiv.className = "comment " + commentCss; - commentDiv.innerHTML = ' \ -
\ - [–] \ - '+comment.author+(isDeleted ? " (deleted by user)" : "")+' \ - '+Format.prettyScore(comment.score)+' point'+((comment.score == 1) ? '': 's')+' \ - '+Format.prettyDate(comment.created_utc)+' \ -
\ -
'+(comment.body === "[removed]" && isRemoved ? "

[likely removed by automoderator]

" : Format.parse(comment.body))+'
\ -
\ - permalink \ - reddit \ -
'; - - return commentDiv; - }; - - return { - createThread: createThread, - createCommentInfo: createCommentInfo, - createCommentSection: createCommentSection, - createComments: createComments - } -})(); +})();*/ if(isSupported) { app.loadPage(); diff --git a/subreddit.html b/subreddit.html index 3c48eda..a8c6d14 100644 --- a/subreddit.html +++ b/subreddit.html @@ -26,7 +26,6 @@ - diff --git a/thread.html b/thread.html index 974411e..a29dbf2 100644 --- a/thread.html +++ b/thread.html @@ -26,7 +26,6 @@ -