diff --git a/dist/index.html b/dist/index.html index 97eb41f..6acc9f7 100644 --- a/dist/index.html +++ b/dist/index.html @@ -21,7 +21,7 @@ -
+
diff --git a/package.json b/package.json index 08579c5..edd93fc 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "scripts": { "start": "webpack-dev-server --mode development", "build": "webpack --mode production", - "build-sass": "sass --style compressed public/sass/index.sass public/main.css" + "build-sass": "sass --style compressed src/sass/index.sass dist/main.css" }, "dependencies": { "lit-html": "^0.9.0", diff --git a/src/api/pushshift/index.js b/src/api/pushshift/index.js index 6138606..1d36b53 100644 --- a/src/api/pushshift/index.js +++ b/src/api/pushshift/index.js @@ -1,4 +1,4 @@ -import { json, toBase10, toBase36 } from 'utils' +import { toBase10, toBase36 } from 'utils' const baseURL = 'https://elastic.pushshift.io' const postURL = `${baseURL}/rs/submissions/_search?source=` @@ -8,16 +8,16 @@ export const getPost = threadID => { const elasticQuery = { query: { term: { - id: toBase10(threadID), - }, - }, + id: toBase10(threadID) + } + } } return ( - fetch(postURL + JSON.stringify(elasticQuery)) - .then(json) - .then(jsonData => jsonData.hits.hits[0]._source) - .then(post => { + fetch(`${postURL}${JSON.stringify(elasticQuery)}`) + .then(response => response.json()) + .then(response => { + const post = response.hits.hits[0]._source post.id = toBase36(post.id) return post }) @@ -41,31 +41,33 @@ export const getComments = threadID => { const elasticQuery = { query: { match: { - link_id: toBase10(threadID), - }, + link_id: toBase10(threadID) + } }, size: 10000, _source: [ - 'author', 'body', 'created_utc', 'parent_id', 'score', 'subreddit', 'link_id', - ], + 'author', 'body', 'created_utc', 'parent_id', 'score', 'subreddit', 'link_id' + ] } return ( fetch(commentURL + JSON.stringify(elasticQuery)) - .then(json) - .then(jsonData => jsonData.hits.hits) - .then(comments => comments.map(comment => { - comment._source.id = toBase36(comment._id) - comment._source.link_id = toBase36(comment._source.link_id) + .then(response => response.json()) + .then(response => { + const comments = response.hits.hits + return comments.map(comment => { + comment._source.id = toBase36(comment._id) + comment._source.link_id = toBase36(comment._source.link_id) - // Missing parent id === direct reply to thread - if (!comment._source.parent_id) { - comment._source.parent_id = threadID - } else { - comment._source.parent_id = toBase36(comment._source.parent_id) - } + // Missing parent id === direct reply to thread + if (!comment._source.parent_id) { + comment._source.parent_id = threadID + } else { + comment._source.parent_id = toBase36(comment._source.parent_id) + } - return comment._source - })) + return comment._source + }) + }) ) } diff --git a/src/api/reddit/auth.js b/src/api/reddit/auth.js index 780a8dc..6ab575a 100644 --- a/src/api/reddit/auth.js +++ b/src/api/reddit/auth.js @@ -7,9 +7,8 @@ const clientID = '33W8M1OOxPv80A' // Token for reddit API let token -// Header for general api calls -export const getAuth = () => { - // We have already gotten a token +const getToken = () => { +// We have already gotten a token if (token !== undefined) { return Promise.resolve(token) } @@ -17,7 +16,7 @@ export const getAuth = () => { // Headers for getting reddit api token const tokenInit = { headers: { - Authorization: `Basic ${btoa(`${clientID}:`)}`, + Authorization: `Basic ${window.btoa(`${clientID}:`)}`, 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8' }, method: 'POST', @@ -30,11 +29,18 @@ export const getAuth = () => { .then(responseBody => { // Save token for later token = responseBody.access_token - return { - headers: { - Authorization: `bearer ${token}` - } - } + return token }) ) } + +// Get header for general api calls +export const getAuth = () => ( + getToken() + .then(token => ({ + headers: { + Authorization: `bearer ${token}` + } + + })) +) diff --git a/src/core/thread.js b/src/core/thread.js index e69de29..17bc68a 100644 --- a/src/core/thread.js +++ b/src/core/thread.js @@ -0,0 +1,74 @@ +import { getPost, getComments as getRedditComments } from 'api/reddit' +import { getPost as getRemovedPost, getComments as getPushshiftComments } from 'api/pushshift' +import { isDeleted, isRemoved } from 'utils' + +export default () => { + const { subreddit, threadID } = this.props.match.params + + Promise.all([ + // Get thread from reddit + getPost(subreddit, threadID) + .then(post => { + this.setState({ post }) + // Fetch the thread from pushshift if it was deleted/removed + if (isDeleted(post.selftext)) { + getRemovedPost(threadID) + .then(removedPost => { + removedPost.removed = true + this.setState({ post: removedPost }) + }) + } + }), + // Get comment ids from pushshift + getPushshiftComments(threadID) + ]) + .then(results => { + const pushshiftComments = results[1] + + // Extract ids from pushshift response + const ids = pushshiftComments.map(comment => comment.id) + console.log('Number of comments from Pushshift:', ids.length) + + // Get all the comments from reddit + return ( + getRedditComments(ids) + .then(redditComments => { + const redditCommentLookup = {} + redditComments.forEach(comment => { + redditCommentLookup[comment.id] = comment + }) + + pushshiftComments.forEach(comment => { + // Replace pushshift score with reddit (its usually more accurate) + const redditComment = redditCommentLookup[comment.id] + if (redditComment !== undefined) { + comment.score = redditComment.score + } + }) + + this.setState({ pushshiftComments }) + return redditComments + }) + ) + }) + .then(redditComments => { + console.log('Number of comments from Reddit:', redditComments.length) + const removed = [] + const deleted = [] + + // Check what as removed / deleted according to reddit + redditComments.forEach(comment => { + if (isRemoved(comment.body)) { + removed.push(comment.id) + } else if (isDeleted(comment.body)) { + deleted.push(comment.id) + } + }) + + this.setState({ + removed, + deleted, + loadingComments: false + }) + }) +} diff --git a/src/index.js b/src/index.js index 62b9854..73bb067 100644 --- a/src/index.js +++ b/src/index.js @@ -1,40 +1,58 @@ -import { render } from 'lit-html/lib/lit-extended' -import Navigo from 'navigo' import aboutTemplate from './templates/about' import threadTemplate from './templates/thread' import subredditTemplate from './templates/subreddit' -import {store} from './state' import getThreads from './core/subreddit' +import SPA from './simpleSPA' -const router = new Navigo(window.location.origin) - -const main = document.getElementById('main') - -// Causes a rerender of the currently selected template -const renderPage = () => { - const template = store.getState().template(store.getState(), store.setState) - render(template, main) +const routes = { + '/about': (_, setState) => { + setState({template: aboutTemplate, title: 'About Removeddit'}) + }, + '/r/:subreddit/comments/:threadID/:junk/:commentID': ({subreddit, threadID, commentID}, setState) => { + setState({template: threadTemplate, subreddit, threadID, commentID}) + }, + '/r/:subreddit/comments/:threadID*': ({subreddit, threadID}, setState) => { + setState({template: threadTemplate, subreddit, threadID, commentID: undefined}) + }, + '/r/:subreddit': ({subreddit}, setState) => { + setState({template: subredditTemplate, threads: [], title: `/r/${subreddit}`, subreddit, threadID: undefined, commentID: undefined}) + getThreads(subreddit, setState) + }, + '*': (_, setState) => { + setState({template: subredditTemplate, threads: [], title: '/r/all', subreddit: 'all', threadID: undefined, commentID: undefined}) + getThreads('all', setState) + } } -// Re-render page whenever state is changed -store.subscribe(() => renderPage()) +const get = (key, defaultValue) => { + const value = localStorage.getItem(key) -router.on({ - '/about': () => { - store.setState({template: aboutTemplate}) - }, - '/r/:subreddit/comments/:threadID/:junk/:commentID': ({subreddit, threadID, commentID}) => { - store.setState({template: threadTemplate, subreddit, threadID, commentID}) - }, - '/r/:subreddit/comments/:threadID*': ({subreddit, threadID}) => { - store.setState({template: threadTemplate, subreddit, threadID, commentID: undefined}) - }, - '/r/:subreddit': ({subreddit}) => { - store.setState({template: subredditTemplate, threads: [], subreddit, threadID: undefined, commentID: undefined}) - getThreads(subreddit, store.setState) - }, - '*': () => { - store.setState({template: subredditTemplate, threads: [], subreddit: 'all', threadID: undefined, commentID: undefined}) - getThreads('all', store.setState) + if (value !== null) { + return JSON.parse(value) } -}).resolve() + + return defaultValue +} + +const initState = { + commentSort: get('commentSort', 'top'), + commentShow: get('commentShow', 'removedDeleted'), + subreddit: undefined, + threadID: undefined, + commentID: undefined, + threads: [], + thread: undefined, + allComments: [], + statusText: undefined, + statusImage: undefined, + title: 'Removeddit' + // comments: () => { + // return store.getState().allComments + // .filter(showFunctions[store.getState().currentShow]) + // .sort(sortFunctions[store.getState().currentSort]) + // } +} + +const persistentState = ['commentSort', 'commentShow'] + +SPA({routes, initState, persistentState, stateLogger: console.log}) diff --git a/src/pages/Subreddit.js b/src/pages/Subreddit.js deleted file mode 100644 index 2bbe50b..0000000 --- a/src/pages/Subreddit.js +++ /dev/null @@ -1,50 +0,0 @@ -import React from 'react' -import { Link } from 'react-router-dom' -import { getRemovedThreadIDs } from '../api/removeddit' -import { getThreads } from '../api/reddit' -import Post from 'components/Post' - -const getSubredditForAPI = props => { - const { subreddit = 'all' } = props.match.params - if (subreddit.toLowerCase() === 'all') { - return '' - } - return subreddit.toLowerCase() -} - -export default class Subreddit extends React.Component { - updateThreads (props) { - const subreddit = getSubredditForAPI(props) - getRemovedThreadIDs(subreddit) - .then(threadIDs => getThreads(threadIDs)) - .then(threads => { - threads.forEach(thread => { - thread.removed = true - thread.selftext = '' - }) - this.setState({ threads, subreddit }) - }) - } - - render () { - const { subreddit = 'all' } = this.props.match.params - const subredditLink = `/r/${subreddit}` - - return ( - -
- {subredditLink} - - reddit - - ceddit -
- { - this.state.threads.map(thread => ( - - )) - } -
- ) - } -} diff --git a/src/sass/common.sass b/src/sass/common.sass index bd328a1..e70c254 100644 --- a/src/sass/common.sass +++ b/src/sass/common.sass @@ -4,7 +4,7 @@ html, body background-color: $background font-family: verdana, arial, helvetica, sans-serif -#main +#root margin: 15px .removed diff --git a/src/sass/index.sass b/src/sass/index.sass index 5f9c58c..4e8c9b8 100644 --- a/src/sass/index.sass +++ b/src/sass/index.sass @@ -1,5 +1,3 @@ - - @import colors @import common @import header diff --git a/src/simpleSPA/index.js b/src/simpleSPA/index.js new file mode 100644 index 0000000..dc3ab77 --- /dev/null +++ b/src/simpleSPA/index.js @@ -0,0 +1,50 @@ +import Navigo from 'navigo' +import { render, html } from 'lit-html/lib/lit-extended' +import { createStore } from './store' + +export default ({ + baseURL = window.location.origin, + hash = '#', + useHash = false, + initState = {}, + persistentState = [], + routes = {}, + stateLogger, + element = document.getElementById('root') +}) => { + const store = createStore({ + template: html``, + ...initState + }, persistentState, stateLogger) + + const router = new Navigo(baseURL, useHash, hash) + + const renderPage = () => { + const {template, title} = store.getState() + + if (typeof template === 'function') { + render(template(store.getState(), store.setState), element) + } else { + // WARNING: html`` might be a function.... + render(template, element) + } + + if (title) { + document.title = title + } + + router.updatePageLinks() + } + // Re-render page whenever state is changed + store.subscribe(() => renderPage()) + + const navigoRoutes = {} + Object.keys(routes).forEach(route => { + navigoRoutes[route] = (params = {}) => routes[route](params, store.setState) + }) + router.on(navigoRoutes).resolve() + + return { + router, store + } +} diff --git a/src/state/store.js b/src/simpleSPA/store.js similarity index 71% rename from src/state/store.js rename to src/simpleSPA/store.js index f30e440..4386819 100644 --- a/src/state/store.js +++ b/src/simpleSPA/store.js @@ -1,6 +1,6 @@ -import {put} from '../utils' +const put = (key, value) => localStorage.setItem(key, JSON.stringify(value)) -export const createStore = (initState = {}, persistentKeys = []) => { +export const createStore = (initState = {}, persistentKeys = [], logger) => { let state = initState let callbacks = [] @@ -18,6 +18,10 @@ export const createStore = (initState = {}, persistentKeys = []) => { ...nextState } + if (typeof logger === 'function') { + logger(state) + } + callbacks.forEach(callback => { callback(state) }) diff --git a/src/state/index.js b/src/state/index.js deleted file mode 100644 index f71b765..0000000 --- a/src/state/index.js +++ /dev/null @@ -1,42 +0,0 @@ -import {createStore} from './store' -import {get, showFunctions, sortFunctions} from '../utils' -import {html} from 'lit-html/lib/lit-extended' - -export const commentSort = { - top: 'top', - bottom: 'bottom', - new: 'new', - old: 'old' -} - -export const commentShow = { - all: 'all', - removedDeleted: 'removedDeleted', - removed: 'removed', - deleted: 'deleted' -} - -export const store = createStore({ - commentSort: get('commentSort', commentSort.top), - commentShow: get('commentShow', commentShow.removedDeleted), - subreddit: undefined, - threadID: undefined, - commentID: undefined, - threads: [], - thread: undefined, - allComments: [], - statusText: undefined, - statusImage: undefined, - template: html``, - comments: () => { - return store.getState().allComments - .filter(showFunctions[store.getState().currentShow]) - .sort(sortFunctions[store.getState().currentSort]) - } -}, ['commentSort', 'commentShow']) - -export const stateImages = { - loading: '/images/loading.gif', - error: '/images/error.png', - success: '/images/done.png' -} diff --git a/src/templates/subreddit/index.js b/src/templates/subreddit/index.js index 94a8ba2..80702f8 100644 --- a/src/templates/subreddit/index.js +++ b/src/templates/subreddit/index.js @@ -1,20 +1,15 @@ import { html } from 'lit-html/lib/lit-extended' import post from '../thread/post' -export default (state, setState) => { - const { subreddit, threads } = state - const subredditLink = `/r/${subreddit}` - - return ( - html` -
- ${subredditLink} - - reddit - - ceddit -
- ${threads.map(thread => post(thread))} +export default ({ subreddit, threads }, setState) => ( + html` +
+ /r/${subreddit} + + reddit + + ceddit +
+ ${threads.map(thread => post(thread))} ` - ) -} +) diff --git a/src/templates/thread/commentInfo.js b/src/templates/thread/commentInfo.js index 850c3ed..3c00b11 100644 --- a/src/templates/thread/commentInfo.js +++ b/src/templates/thread/commentInfo.js @@ -1,15 +1,15 @@ -import {html} from 'lit-html' +import {html} from 'lit-html/lib/lit-extended' const getProcent = (part, total) => (total === 0 ? '0.0' : ((100 * part) / total).toFixed(1)) -export default (props) => html` +export default ({ removed, deleted, total }) => html`
- removed comments: ${props.removed}/${props.total} (${getProcent(props.removed, props.total)}%) + removed comments: ${removed}/${total} (${getProcent(removed, total)}%)
- deleted comments: ${props.deleted}/${props.total} (${getProcent(props.deleted, props.total)}%) + deleted comments: ${deleted}/${total} (${getProcent(deleted, total)}%)
` diff --git a/src/templates/thread/post.js b/src/templates/thread/post.js index 1a1bf97..953f826 100644 --- a/src/templates/thread/post.js +++ b/src/templates/thread/post.js @@ -25,7 +25,7 @@ export default (props) => { } return html` -
+
${props.position && html`${props.position}`}
diff --git a/src/utils/index.js b/src/utils/index.js index 7453377..416b241 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -23,9 +23,6 @@ export const chunk = (arr, size) => { return chunks } -// JSON parsing for fetch -export const json = response => response.json() - // Make multiple requests to the same url, with an array of data (usually comment IDs) // This is needed since there is a limit on how long a url can be export const fetchMultiple = (url, arr, header, size = 100) => { @@ -34,7 +31,7 @@ export const fetchMultiple = (url, arr, header, size = 100) => { return Promise.all(subArrays.map(subArr => fetch(url + subArr.join(), header))) } -export const jsonMultiple = responses => Promise.all(responses.map(json)) +export const jsonMultiple = responses => Promise.all(responses.map(resp => resp.json())) // Change bases export const toBase36 = number => parseInt(number, 10).toString(36) @@ -84,13 +81,6 @@ export function prettyScore (score) { return score } -// Retrieve, store and delete stuff in the local storage -export const get = (key, defaultValue) => ( - localStorage.getItem(key) !== null ? JSON.parse(localStorage.getItem(key)) : defaultValue -) - -export const put = (key, value) => localStorage.setItem(key, JSON.stringify(value)) - // Filter comments export const showFunctions = { all: comment => true,