From 67c485786364e3db5eb4cb54bd5c509569de1216 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:54:49 +0000 Subject: [PATCH 01/78] Bump graphql from 16.6.0 to 16.8.1 in /apple/gql-server Bumps [graphql](https://github.com/graphql/graphql-js) from 16.6.0 to 16.8.1. - [Release notes](https://github.com/graphql/graphql-js/releases) - [Commits](https://github.com/graphql/graphql-js/compare/v16.6.0...v16.8.1) --- updated-dependencies: - dependency-name: graphql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- apple/gql-server/package.json | 2 +- apple/gql-server/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apple/gql-server/package.json b/apple/gql-server/package.json index 3337f3b4b..9088164b4 100644 --- a/apple/gql-server/package.json +++ b/apple/gql-server/package.json @@ -9,6 +9,6 @@ "dependencies": { "express": "^4.18.1", "express-graphql": "^0.12.0", - "graphql": "^16.4.0" + "graphql": "^16.8.1" } } \ No newline at end of file diff --git a/apple/gql-server/yarn.lock b/apple/gql-server/yarn.lock index ae72deb0f..587c08d1d 100644 --- a/apple/gql-server/yarn.lock +++ b/apple/gql-server/yarn.lock @@ -194,10 +194,10 @@ get-intrinsic@^1.0.2: has "^1.0.3" has-symbols "^1.0.3" -graphql@^16.4.0: - version "16.6.0" - resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.6.0.tgz#c2dcffa4649db149f6282af726c8c83f1c7c5fdb" - integrity sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw== +graphql@^16.8.1: + version "16.8.1" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.8.1.tgz#1930a965bef1170603702acdb68aedd3f3cf6f07" + integrity sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw== has-symbols@^1.0.3: version "1.0.3" From 8f24f17da87201bf616b7b5ab6b36703ed502fe6 Mon Sep 17 00:00:00 2001 From: Remy Chantenay Date: Thu, 12 Oct 2023 08:36:33 +0200 Subject: [PATCH 02/78] Add label name length validation upon creation --- .../ui/components/LabelsSelectionSheet.kt | 29 +++++++++++---- .../omnivore/ui/components/LabelsViewModel.kt | 37 ++++++++++--------- .../app/src/main/res/values/strings.xml | 1 + 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt index 0461149a7..7022e28f8 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt @@ -3,6 +3,7 @@ package app.omnivore.omnivore.ui.components import LabelChip +import android.widget.Toast import androidx.compose.foundation.* import androidx.compose.foundation.interaction.FocusInteraction import androidx.compose.foundation.interaction.MutableInteractionSource @@ -341,19 +342,33 @@ fun LabelsSelectionSheetContent( ) if (!isLibraryMode && filterTextValue.text.isNotEmpty() && currentLabel == null) { + val context = LocalContext.current Row( horizontalArrangement = Arrangement.Start, verticalAlignment = Alignment.CenterVertically, modifier = Modifier .fillMaxWidth() .clickable { - val label = findOrCreateLabel( - labelsViewModel = labelsViewModel, - labels = labels, - name = filterTextValue - ) - state.addChip(LabelChipView(label)) - filterTextValue = TextFieldValue() + when(labelsViewModel.validateLabelName(filterTextValue.text)) { + LabelsViewModel.Error.LabelNameTooLong -> { + Toast.makeText( + context, + context.getString(R.string.label_selection_sheet_label_too_long_error_msg, + labelsViewModel.labelNameMaxLength), + Toast.LENGTH_SHORT + ).show() + } + null -> { + val label = findOrCreateLabel( + labelsViewModel = labelsViewModel, + labels = labels, + name = filterTextValue + ) + + state.addChip(LabelChipView(label)) + filterTextValue = TextFieldValue() + } + } } .padding(horizontal = 10.dp) .padding(top = 10.dp, bottom = 5.dp) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt index f75959f70..7b26a05df 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt @@ -1,34 +1,16 @@ package app.omnivore.omnivore.ui.components - -import android.content.ClipData -import android.content.ClipboardManager -import android.content.Context -import android.content.Intent -import android.net.Uri import android.util.Log -import android.widget.Toast -import androidx.compose.ui.platform.LocalContext -import androidx.core.content.ContextCompat.startActivity import androidx.lifecycle.* -import app.omnivore.omnivore.DatastoreKeys import app.omnivore.omnivore.DatastoreRepository import app.omnivore.omnivore.dataService.* import app.omnivore.omnivore.graphql.generated.type.CreateLabelInput -import app.omnivore.omnivore.graphql.generated.type.SetLabelsInput import app.omnivore.omnivore.models.ServerSyncStatus import app.omnivore.omnivore.networking.* -import app.omnivore.omnivore.persistence.entities.SavedItem -import app.omnivore.omnivore.persistence.entities.SavedItemAndSavedItemLabelCrossRef import app.omnivore.omnivore.persistence.entities.SavedItemLabel -import app.omnivore.omnivore.ui.components.LabelSwatchHelper -import app.omnivore.omnivore.ui.library.SavedItemAction import com.apollographql.apollo3.api.Optional.Companion.presentIfNotNull -import com.google.gson.Gson import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.* -import kotlinx.coroutines.flow.distinctUntilChanged -import java.time.Instant import java.time.LocalDate import java.time.ZoneOffset import java.time.format.DateTimeFormatter @@ -42,6 +24,25 @@ class LabelsViewModel @Inject constructor( private val dataService: DataService, private val networker: Networker ): ViewModel() { + val labelNameMaxLength = 64 + + enum class Error { + LabelNameTooLong + } + + /** + * Checks whether or not the provided label name is valid. + * @param labelName The name of the label. + * @return null if valid, [Error] otherwise. + */ + fun validateLabelName(labelName: String): Error? { + val trimmedName = labelName.trim() + if (trimmedName.count() > labelNameMaxLength) { + return Error.LabelNameTooLong + } + + return null + } fun createNewSavedItemLabelWithTemp(labelName: String, hexColorValue: String): SavedItemLabel { val tempId = UUID.randomUUID().toString() diff --git a/android/Omnivore/app/src/main/res/values/strings.xml b/android/Omnivore/app/src/main/res/values/strings.xml index ad7f2a3ea..2867be8e3 100644 --- a/android/Omnivore/app/src/main/res/values/strings.xml +++ b/android/Omnivore/app/src/main/res/values/strings.xml @@ -103,6 +103,7 @@ Search Save Create a new label named \"%1$s\" + The name provided is too long (must be less or equal than %1$d characters) Labels From 4ae6c476f819b27ec85e168b44e85bb988c3960e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sat, 14 Oct 2023 13:18:40 +0800 Subject: [PATCH 03/78] shows api error message --- packages/rule-handler/src/filter.ts | 13 ++++++++++--- packages/rule-handler/src/rule.ts | 1 - 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/rule-handler/src/filter.ts b/packages/rule-handler/src/filter.ts index a3c43e902..2eb365c05 100644 --- a/packages/rule-handler/src/filter.ts +++ b/packages/rule-handler/src/filter.ts @@ -6,6 +6,9 @@ interface SearchResponse { edges: Edge[] } } + errors?: { + message: string + }[] } interface Edge { @@ -28,7 +31,6 @@ interface Label { } export const search = async ( - userId: string, apiEndpoint: string, auth: string, query: string @@ -75,6 +77,12 @@ export const search = async ( } ) + if (response.data.errors) { + console.error(response.data.errors) + + return [] + } + const edges = response.data.data.search.edges if (edges.length === 0) { return [] @@ -89,14 +97,13 @@ export const search = async ( } export const filterPage = async ( - userId: string, apiEndpoint: string, auth: string, filter: string, pageId: string ): Promise => { filter += ` includes:${pageId}` - const pages = await search(userId, apiEndpoint, auth, filter) + const pages = await search(apiEndpoint, auth, filter) return pages.length > 0 ? pages[0] : null } diff --git a/packages/rule-handler/src/rule.ts b/packages/rule-handler/src/rule.ts index b7345fb2c..ef80aa973 100644 --- a/packages/rule-handler/src/rule.ts +++ b/packages/rule-handler/src/rule.ts @@ -93,7 +93,6 @@ export const triggerActions = async ( } const filteredPage = await filterPage( - userId, apiEndpoint, authToken, rule.filter, From 668485ba15b75b04a76b34942e707b5bcd31d881 Mon Sep 17 00:00:00 2001 From: Remy Chantenay Date: Sat, 14 Oct 2023 12:57:35 +0200 Subject: [PATCH 04/78] Trim labelname upon creation --- .../ui/components/LabelsSelectionSheet.kt | 15 ++++++++------- .../omnivore/ui/components/LabelsViewModel.kt | 3 +-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt index 7022e28f8..7dc98a9f7 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt @@ -220,12 +220,12 @@ class LabelChipView(label: SavedItemLabel) : Chip(label.name) { val label = label } -fun findOrCreateLabel(labelsViewModel: LabelsViewModel, labels: List, name: TextFieldValue): SavedItemLabel { - val found = labels.find { it.name == name.text } +fun findOrCreateLabel(labelsViewModel: LabelsViewModel, labels: List, name: String): SavedItemLabel { + val found = labels.find { it.name == name } if (found != null) { return found } - return labelsViewModel.createNewSavedItemLabelWithTemp(name.text, LabelSwatchHelper.random()) + return labelsViewModel.createNewSavedItemLabelWithTemp(name, LabelSwatchHelper.random()) } @Composable @@ -309,7 +309,7 @@ fun LabelsSelectionSheetContent( LabelChipView(it) } ?: null } else { - LabelChipView(findOrCreateLabel(labelsViewModel = labelsViewModel, labels = labels, name = it)) + LabelChipView(findOrCreateLabel(labelsViewModel = labelsViewModel, labels = labels, name = it.text)) } }, chipLeadingIcon = { chip -> CircleIcon(colorHex = chip.label.color) }, @@ -349,7 +349,8 @@ fun LabelsSelectionSheetContent( modifier = Modifier .fillMaxWidth() .clickable { - when(labelsViewModel.validateLabelName(filterTextValue.text)) { + val labelName = filterTextValue.text.trim() + when(labelsViewModel.validateLabelName(labelName)) { LabelsViewModel.Error.LabelNameTooLong -> { Toast.makeText( context, @@ -362,7 +363,7 @@ fun LabelsSelectionSheetContent( val label = findOrCreateLabel( labelsViewModel = labelsViewModel, labels = labels, - name = filterTextValue + name = labelName ) state.addChip(LabelChipView(label)) @@ -379,7 +380,7 @@ fun LabelsSelectionSheetContent( contentDescription = null, modifier = Modifier.padding(end = 8.dp) ) - Text(text = stringResource(R.string.label_selection_sheet_text_create, filterTextValue.text)) + Text(text = stringResource(R.string.label_selection_sheet_text_create, filterTextValue.text.trim())) } } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt index 7b26a05df..ddcdaf07b 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt @@ -36,8 +36,7 @@ class LabelsViewModel @Inject constructor( * @return null if valid, [Error] otherwise. */ fun validateLabelName(labelName: String): Error? { - val trimmedName = labelName.trim() - if (trimmedName.count() > labelNameMaxLength) { + if (labelName.count() > labelNameMaxLength) { return Error.LabelNameTooLong } From 34798f39f03223c3eec55f5dcbb6d0df7cb41df3 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 11:25:37 +0800 Subject: [PATCH 05/78] Disable JSONLD fetching which can be quite slow --- packages/api/src/utils/parser.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index f8ca6ffa2..ae047ef82 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -332,18 +332,13 @@ export const parsePreparedContent = async ( DOMPurify.addHook('uponSanitizeElement', domPurifySanitizeHook) const clean = DOMPurify.sanitize(article?.content || '', DOM_PURIFY_CONFIG) - const jsonLdLinkMetadata = (async () => { - return getJSONLdLinkMetadata(dom) - })() - Object.assign(article || {}, { content: clean, - title: article?.title || (await jsonLdLinkMetadata).title, - previewImage: - article?.previewImage || (await jsonLdLinkMetadata).previewImage, - siteName: article?.siteName || (await jsonLdLinkMetadata).siteName, + title: article?.title, + previewImage: article?.previewImage, + siteName: article?.siteName, siteIcon: article?.siteIcon, - byline: article?.byline || (await jsonLdLinkMetadata).byline, + byline: article?.byline, language: article?.language, }) logRecord.parseSuccess = true From 4f86f05aac8ff02186ed40dffabbe28ce91856fc Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 11:39:03 +0800 Subject: [PATCH 06/78] Remove JSONLD test --- packages/api/test/utils/parser.test.ts | 50 +++++++++++++------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/api/test/utils/parser.test.ts b/packages/api/test/utils/parser.test.ts index 4dfa720d0..939a1d17c 100644 --- a/packages/api/test/utils/parser.test.ts +++ b/packages/api/test/utils/parser.test.ts @@ -78,32 +78,32 @@ describe('parsePreparedContent', () => { }) }) -describe('parsePreparedContent', () => { - nock('https://oembeddata').get('/').reply(200, { - version: '1.0', - provider_name: 'Hippocratic Adventures', - provider_url: 'https://www.hippocraticadventures.com', - title: - 'The Ultimate Guide to Practicing Medicine in Singapore – Part 2', - }) +// describe('parsePreparedContent', () => { +// nock('https://oembeddata').get('/').reply(200, { +// version: '1.0', +// provider_name: 'Hippocratic Adventures', +// provider_url: 'https://www.hippocraticadventures.com', +// title: +// 'The Ultimate Guide to Practicing Medicine in Singapore – Part 2', +// }) - it('gets metadata from external JSONLD if available', async () => { - const html = ` - - - - body - ` - const result = await parsePreparedContent('https://blog.omnivore.app/', { - document: html, - pageInfo: {}, - }) - expect(result.parsedContent?.title).to.equal( - 'The Ultimate Guide to Practicing Medicine in Singapore – Part 2' - ) - }) -}) +// it('gets metadata from external JSONLD if available', async () => { +// const html = ` +// +// +// +// body +// ` +// const result = await parsePreparedContent('https://blog.omnivore.app/', { +// document: html, +// pageInfo: {}, +// }) +// expect(result.parsedContent?.title).to.equal( +// 'The Ultimate Guide to Practicing Medicine in Singapore – Part 2' +// ) +// }) +// }) describe('isProbablyArticle', () => { let user: User From f1a71ae86c9ec98291f6a138bfb188e30a8fb876 Mon Sep 17 00:00:00 2001 From: Remy Chantenay Date: Mon, 16 Oct 2023 09:05:56 +0200 Subject: [PATCH 07/78] Android: Show URL in OpenLinkView --- .../omnivore/ui/reader/OpenLinkView.kt | 59 +++++++++++++++++++ .../ui/reader/WebReaderLoadingContainer.kt | 36 ----------- 2 files changed, 59 insertions(+), 36 deletions(-) create mode 100644 android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt new file mode 100644 index 000000000..4ac6f8545 --- /dev/null +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt @@ -0,0 +1,59 @@ +package app.omnivore.omnivore.ui.reader + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Button +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import app.omnivore.omnivore.R + +@Composable +fun OpenLinkView(webReaderViewModel: WebReaderViewModel) { + val context = LocalContext.current + + Column(modifier = Modifier + .padding(top = 25.dp) + .padding(horizontal = 50.dp), verticalArrangement = Arrangement.spacedBy(20.dp)) { + Row { + Text(webReaderViewModel.currentLink.toString(), + fontWeight = FontWeight.Light, + color = Color.DarkGray, + fontSize = 12.sp, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + Row(modifier = Modifier.padding(top = 25.dp)) { + Button(onClick = { webReaderViewModel.openCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_open_in_browser)) + } + } + Row { + Button(onClick = { webReaderViewModel.saveCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_save_to_omnivore)) + + } + } + Row { + Button(onClick = {webReaderViewModel.copyCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_copy_link)) + } + } + Row { + Button(onClick = {webReaderViewModel.resetBottomSheet() }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_cancel)) + } + } + } +} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt index e680a42a7..87a0d63ed 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt @@ -42,7 +42,6 @@ import com.google.accompanist.systemuicontroller.rememberSystemUiController import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.launch import kotlin.math.roundToInt -import androidx.compose.material3.Button import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import app.omnivore.omnivore.ui.components.LabelsViewModel @@ -450,38 +449,3 @@ fun BottomSheetUI(title: String?, content: @Composable () -> Unit) { } } } - -@OptIn(ExperimentalMaterial3Api::class, ExperimentalMaterialApi::class) -@Composable -fun OpenLinkView(webReaderViewModel: WebReaderViewModel) { - val context = LocalContext.current - - Column(modifier = Modifier - .padding(top = 50.dp) - .padding(horizontal = 50.dp), verticalArrangement = Arrangement.spacedBy(20.dp)) { - Row { - Button(onClick = { webReaderViewModel.openCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_open_in_browser)) - - } - } - Row() { - Button(onClick = { webReaderViewModel.saveCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_save_to_omnivore)) - - } - } - Row() { - Button(onClick = {webReaderViewModel.copyCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_copy_link)) - - } - } - Row { - Button(onClick = {webReaderViewModel.resetBottomSheet() }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_cancel)) - - } - } - } -} From e54c1c81a163ae503b73f69db0955724533884f6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 16 Oct 2023 15:24:53 +0800 Subject: [PATCH 08/78] soft delete user by calling delete account api --- packages/api/src/entity/user.ts | 1 + packages/api/src/resolvers/user/index.ts | 14 +++-- packages/api/src/services/user.ts | 2 +- packages/api/test/resolvers/user.test.ts | 56 ++++++++++++++++++- .../0135.do.alter_user_status_type.sql | 9 +++ .../0135.undo.alter_user_status_type.sql | 9 +++ 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100755 packages/db/migrations/0135.do.alter_user_status_type.sql create mode 100755 packages/db/migrations/0135.undo.alter_user_status_type.sql diff --git a/packages/api/src/entity/user.ts b/packages/api/src/entity/user.ts index a926b5a89..83526f946 100644 --- a/packages/api/src/entity/user.ts +++ b/packages/api/src/entity/user.ts @@ -22,6 +22,7 @@ export enum RegistrationType { export enum StatusType { Active = 'ACTIVE', Pending = 'PENDING', + Deleted = 'DELETED', } @Entity() diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index 8b80e1340..bf99a5c04 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -1,5 +1,9 @@ import * as jwt from 'jsonwebtoken' -import { RegistrationType, User as UserEntity } from '../../entity/user' +import { + RegistrationType, + StatusType, + User as UserEntity, +} from '../../entity/user' import { env } from '../../env' import { DeleteAccountError, @@ -38,6 +42,7 @@ import { import { userRepository } from '../../repository/user' import { createUser } from '../../services/create_user' import { sendVerificationEmail } from '../../services/send_emails' +import { updateUser } from '../../services/user' import { authorized, userDataToUser } from '../../utils/helpers' import { validateUsername } from '../../utils/usernamePolicy' import { WithDataSourcesContext } from '../types' @@ -313,9 +318,10 @@ export const deleteAccountResolver = authorized< DeleteAccountSuccess, DeleteAccountError, MutationDeleteAccountArgs ->(async (_, { userID }, { authTrx, log }) => { - const result = await authTrx(async (t) => { - return t.withRepository(userRepository).delete(userID) +>(async (_, { userID }, { log }) => { + // soft delete user + const result = await updateUser(userID, { + status: StatusType.Deleted, }) if (!result.affected) { log.error('Error deleting user account') diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index aed61b8f9..39babc270 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -13,7 +13,7 @@ export const deleteUser = async (userId: string) => { } export const updateUser = async (userId: string, update: Partial) => { - await authTrx( + return authTrx( async (t) => t.getRepository(User).update(userId, update), undefined, userId diff --git a/packages/api/test/resolvers/user.test.ts b/packages/api/test/resolvers/user.test.ts index 404f4a09d..82e0ceefc 100644 --- a/packages/api/test/resolvers/user.test.ts +++ b/packages/api/test/resolvers/user.test.ts @@ -9,7 +9,7 @@ import { findProfile } from '../../src/services/profile' import { deleteUser, findUser } from '../../src/services/user' import { hashPassword } from '../../src/utils/auth' import { createTestUser } from '../db' -import { graphqlRequest, request } from '../util' +import { generateFakeUuid, graphqlRequest, request } from '../util' describe('User API', () => { const correctPassword = 'fakePassword' @@ -238,4 +238,58 @@ describe('User API', () => { return graphqlRequest(query, invalidAuthToken).expect(500) }) }) + + describe('Delete account', () => { + const query = (userId: string) => ` + mutation { + deleteAccount( + userID: "${userId}" + ) { + ... on DeleteAccountSuccess { + userID + } + ... on DeleteAccountError { + errorCodes + } + } + } + ` + + let userId: string + let authToken: string + + before(async () => { + const user = await createTestUser('to_delete_user') + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + userId = user.id + authToken = res.body.authToken + }) + + after(async () => { + await deleteUser(userId) + }) + + context('when user id is valid', () => { + it('deletes user and responds with 200', async () => { + const response = await graphqlRequest(query(userId), authToken).expect( + 200 + ) + expect(response.body.data.deleteAccount.userID).to.eql(userId) + }) + }) + + context('when user not found', () => { + it('responds with error code UserNotFound', async () => { + const response = await graphqlRequest( + query(generateFakeUuid()), + authToken + ).expect(200) + expect(response.body.data.deleteAccount.errorCodes).to.eql([ + 'USER_NOT_FOUND', + ]) + }) + }) + }) }) diff --git a/packages/db/migrations/0135.do.alter_user_status_type.sql b/packages/db/migrations/0135.do.alter_user_status_type.sql new file mode 100755 index 000000000..da4f39eef --- /dev/null +++ b/packages/db/migrations/0135.do.alter_user_status_type.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: alter_user_status_type +-- Description: Add DELETED to the user_status_type enum + +BEGIN; + +ALTER TYPE user_status_type ADD VALUE 'DELETED'; + +COMMIT; diff --git a/packages/db/migrations/0135.undo.alter_user_status_type.sql b/packages/db/migrations/0135.undo.alter_user_status_type.sql new file mode 100755 index 000000000..a39f1f7ba --- /dev/null +++ b/packages/db/migrations/0135.undo.alter_user_status_type.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: alter_user_status_type +-- Description: Add DELETED to the user_status_type enum + +BEGIN; + +ALTER TYPE user_status_type DROP VALUE IF EXISTS 'DELETED'; + +COMMIT; From 9e8ccb35e3dca496b8abd217dd857e6af7ee53b5 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 15:48:03 +0800 Subject: [PATCH 09/78] Limit users to a single import a day --- .../api/src/resolvers/importers/uploadImportFileResolver.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts index 4f327ce54..4feb518de 100644 --- a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -16,7 +16,7 @@ import { generateUploadSignedUrl, } from '../../utils/uploads' -const MAX_DAILY_UPLOADS = 4 +const MAX_DAILY_UPLOADS = 1 const VALID_CONTENT_TYPES = ['text/csv', 'application/zip'] const extensionForContentType = (contentType: string) => { @@ -61,7 +61,7 @@ export const uploadImportFileResolver = authorized< const dirPath = `imports/${uid}/${dateStr}/` const fileCount = await countOfFilesWithPrefix(dirPath) - if (fileCount > MAX_DAILY_UPLOADS) { + if (fileCount >= MAX_DAILY_UPLOADS) { return { errorCodes: [UploadImportFileErrorCode.UploadDailyLimitExceeded], } From 00bd1832876ab82f375c7df0aba0bfe9a6fa6dcc Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 16 Oct 2023 16:33:22 +0800 Subject: [PATCH 10/78] do not retry importer job if user account is deleted --- packages/puppeteer-parse/index.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 11726b9ae..b3413a244 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -289,6 +289,10 @@ const sendSavePageMutation = async (userId, input) => { if (response.data.data.savePage.errorCodes && response.data.data.savePage.errorCodes.length > 0) { console.error('error while saving page', response.data.data.savePage.errorCodes[0]); + if (response.data.data.savePage.errorCodes[0] === 'UNAUTHORIZED') { + return { error: 'UNAUTHORIZED' }; + } + return null; } @@ -473,6 +477,9 @@ async function fetchContent(req, res) { if (!apiResponse) { logRecord.error = 'error while saving page'; statusCode = 500; + } else if (apiResponse.error === 'UNAUTHORIZED') { + console.info('user is deleted, do not retry', logRecord); + return res.sendStatus(200); } else { importStatus = readabilityResult ? 'imported' : 'failed'; } From 1b2d93e11842c87103c2ec00f093952f5f7f93a7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 16 Oct 2023 17:35:29 +0800 Subject: [PATCH 11/78] remove duplicate rss subscriptions by user and url; create a unique constraint --- packages/api/src/repository/user.ts | 4 +-- .../importers/uploadImportFileResolver.ts | 4 +-- .../src/resolvers/recommendations/index.ts | 8 ++---- packages/api/src/resolvers/save/index.ts | 26 ++++++------------ .../send_install_instructions/index.ts | 6 ++--- .../api/src/resolvers/subscriptions/index.ts | 3 ++- packages/api/src/resolvers/user/index.ts | 27 +++++++++---------- packages/api/src/routers/auth/auth_router.ts | 4 +-- .../api/src/routers/auth/mobile/sign_in.ts | 2 +- packages/api/src/routers/user_router.ts | 2 +- packages/api/src/services/newsletters.ts | 5 +--- packages/api/src/services/save_url.ts | 4 +-- packages/api/src/services/user.ts | 4 +-- packages/api/src/utils/parser.ts | 2 ++ .../api/test/resolvers/subscriptions.test.ts | 2 +- ...0136.do.add_unique_to_subscription_url.sql | 19 +++++++++++++ ...36.undo.add_unique_to_subscription_url.sql | 9 +++++++ 17 files changed, 70 insertions(+), 61 deletions(-) create mode 100755 packages/db/migrations/0136.do.add_unique_to_subscription_url.sql create mode 100755 packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql diff --git a/packages/api/src/repository/user.ts b/packages/api/src/repository/user.ts index 408543c95..a8f327b0d 100644 --- a/packages/api/src/repository/user.ts +++ b/packages/api/src/repository/user.ts @@ -1,6 +1,6 @@ import { In } from 'typeorm' import { appDataSource } from '../data_source' -import { User } from './../entity/user' +import { StatusType, User } from './../entity/user' const TOP_USERS = [ 'jacksonh', @@ -16,7 +16,7 @@ export const MAX_RECORDS_LIMIT = 1000 export const userRepository = appDataSource.getRepository(User).extend({ findById(id: string) { - return this.findOneBy({ id }) + return this.findOneBy({ id, status: StatusType.Active }) }, findByEmail(email: string) { diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts index 4f327ce54..efa88f9b9 100644 --- a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -33,14 +33,14 @@ export const uploadImportFileResolver = authorized< UploadImportFileSuccess, UploadImportFileError, MutationUploadImportFileArgs ->(async (_, { type, contentType }, { claims: { uid }, log }) => { +>(async (_, { type, contentType }, { uid }) => { if (!VALID_CONTENT_TYPES.includes(contentType)) { return { errorCodes: [UploadImportFileErrorCode.BadRequest], } } - const user = await userRepository.findOneBy({ id: uid }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [UploadImportFileErrorCode.Unauthorized], diff --git a/packages/api/src/resolvers/recommendations/index.ts b/packages/api/src/resolvers/recommendations/index.ts index d3062dc5d..31f7da898 100644 --- a/packages/api/src/resolvers/recommendations/index.ts +++ b/packages/api/src/resolvers/recommendations/index.ts @@ -48,9 +48,7 @@ export const createGroupResolver = authorized< MutationCreateGroupArgs >(async (_, { input }, { uid, log }) => { try { - const userData = await userRepository.findOneBy({ - id: uid, - }) + const userData = await userRepository.findById(uid) if (!userData) { return { errorCodes: [CreateGroupErrorCode.Unauthorized], @@ -107,9 +105,7 @@ export const createGroupResolver = authorized< export const groupsResolver = authorized( async (_, __, { uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [GroupsErrorCode.Unauthorized], diff --git a/packages/api/src/resolvers/save/index.ts b/packages/api/src/resolvers/save/index.ts index 2acf0ce3b..f5508663d 100644 --- a/packages/api/src/resolvers/save/index.ts +++ b/packages/api/src/resolvers/save/index.ts @@ -18,9 +18,9 @@ export const savePageResolver = authorized< SaveSuccess, SaveError, MutationSavePageArgs ->(async (_, { input }, ctx) => { +>(async (_, { input }, { uid }) => { analytics.track({ - userId: ctx.uid, + userId: uid, event: 'link_saved', properties: { url: input.url, @@ -30,9 +30,7 @@ export const savePageResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: ctx.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } @@ -44,11 +42,7 @@ export const saveUrlResolver = authorized< SaveSuccess, SaveError, MutationSaveUrlArgs ->(async (_, { input }, ctx) => { - const { - claims: { uid }, - } = ctx - +>(async (_, { input }, { uid }) => { analytics.track({ userId: uid, event: 'link_saved', @@ -60,9 +54,7 @@ export const saveUrlResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } @@ -74,9 +66,9 @@ export const saveFileResolver = authorized< SaveSuccess, SaveError, MutationSaveFileArgs ->(async (_, { input }, ctx) => { +>(async (_, { input }, { uid }) => { analytics.track({ - userId: ctx.uid, + userId: uid, event: 'link_saved', properties: { url: input.url, @@ -86,9 +78,7 @@ export const saveFileResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: ctx.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } diff --git a/packages/api/src/resolvers/send_install_instructions/index.ts b/packages/api/src/resolvers/send_install_instructions/index.ts index 6435b807b..62d134fbc 100644 --- a/packages/api/src/resolvers/send_install_instructions/index.ts +++ b/packages/api/src/resolvers/send_install_instructions/index.ts @@ -14,11 +14,9 @@ const INSTALL_INSTRUCTIONS_EMAIL_TEMPLATE_ID = export const sendInstallInstructionsResolver = authorized< SendInstallInstructionsSuccess, SendInstallInstructionsError ->(async (_parent, _args, { claims, log }) => { +>(async (_parent, _args, { uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: claims.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SendInstallInstructionsErrorCode.Unauthorized] } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index db249b9b9..7076e8524 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -89,7 +89,8 @@ export const subscriptionsResolver = authorized< } const subscriptions = await queryBuilder - .orderBy(`subscription.${sortBy}`, sortOrder, 'NULLS LAST') + .orderBy('subscription.status', 'ASC') + .addOrderBy(`subscription.${sortBy}`, sortOrder, 'NULLS LAST') .getMany() return { diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index bf99a5c04..38b0111e2 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -52,9 +52,7 @@ export const updateUserResolver = authorized< UpdateUserError, MutationUpdateUserArgs >(async (_, { input: { name, bio } }, { uid, authTrx }) => { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [UpdateUserErrorCode.UserNotFound] } } @@ -92,9 +90,7 @@ export const updateUserProfileResolver = authorized< UpdateUserProfileError, MutationUpdateUserProfileArgs >(async (_, { input: { userId, username, pictureUrl } }, { uid, authTrx }) => { - const user = await userRepository.findOneBy({ - id: userId, - }) + const user = await userRepository.findById(userId) if (!user) { return { errorCodes: [UpdateUserProfileErrorCode.Unauthorized] } } @@ -117,6 +113,7 @@ export const updateUserProfileResolver = authorized< profile: { username: lowerCasedUsername, }, + status: StatusType.Active, }) if (existingUser?.id) { return { @@ -161,6 +158,7 @@ export const googleLoginResolver: ResolverFn< const user = await userRepository.findOneBy({ email, + status: StatusType.Active, }) if (!user?.id) { return { errorCodes: [LoginErrorCode.UserNotFound] } @@ -256,9 +254,7 @@ export const getMeUserResolver: ResolverFn< return undefined } - const user = await userRepository.findOneBy({ - id: claims.uid, - }) + const user = await userRepository.findById(claims.uid) if (!user) { return undefined } @@ -282,12 +278,17 @@ export const getUserResolver: ResolverFn< const userId = id || (username && - (await userRepository.findOneBy({ profile: { username } }))?.id) + ( + await userRepository.findOneBy({ + profile: { username }, + status: StatusType.Active, + }) + )?.id) if (!userId) { return { errorCodes: [UserErrorCode.UserNotFound] } } - const userRecord = await userRepository.findOneBy({ id: userId }) + const userRecord = await userRepository.findById(userId) if (!userRecord) { return { errorCodes: [UserErrorCode.UserNotFound] } } @@ -340,9 +341,7 @@ export const updateEmailResolver = authorized< MutationUpdateEmailArgs >(async (_, { input: { email } }, { authTrx, uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 366c10ada..333391d62 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -421,7 +421,7 @@ export function authRouter() { const { email, password } = req.body try { const user = await userRepository.findByEmail(email.trim()) - if (!user?.id) { + if (!user || user.status === StatusType.Deleted) { return res.redirect( `${env.client.url}/auth/email-login?errorCodes=${LoginErrorCode.UserNotFound}` ) @@ -610,7 +610,7 @@ export function authRouter() { try { const user = await userRepository.findByEmail(email) - if (!user) { + if (!user || user.status === StatusType.Deleted) { return res.redirect(`${env.client.url}/auth/reset-sent`) } diff --git a/packages/api/src/routers/auth/mobile/sign_in.ts b/packages/api/src/routers/auth/mobile/sign_in.ts index 40d4462bb..4af80f0a6 100644 --- a/packages/api/src/routers/auth/mobile/sign_in.ts +++ b/packages/api/src/routers/auth/mobile/sign_in.ts @@ -46,7 +46,7 @@ export async function createMobileEmailSignInResponse( } const user = await userRepository.findByEmail(email.trim()) - if (!user?.id || !user?.password) { + if (!user || !user.password || user.status === StatusType.Deleted) { throw new Error('user not found') } diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts index cc830fc71..8e13ee184 100644 --- a/packages/api/src/routers/user_router.ts +++ b/packages/api/src/routers/user_router.ts @@ -39,7 +39,7 @@ export function userRouter() { return } try { - const user = await userRepository.findOneBy({ id: claims.uid }) + const user = await userRepository.findById(claims.uid) if (!user) { res.status(400).send('Bad Request') return diff --git a/packages/api/src/services/newsletters.ts b/packages/api/src/services/newsletters.ts index fb9e33cca..f492cf4b5 100644 --- a/packages/api/src/services/newsletters.ts +++ b/packages/api/src/services/newsletters.ts @@ -21,10 +21,7 @@ export const createNewsletterEmail = async ( userId: string, confirmationCode?: string ): Promise => { - const user = await userRepository.findOne({ - where: { id: userId }, - relations: ['profile'], - }) + const user = await userRepository.findById(userId) if (!user) { return Promise.reject({ errorCode: CreateNewsletterEmailErrorCode.Unauthorized, diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index 0f14b1658..3165bb8fb 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -42,9 +42,7 @@ export const saveUrlFromEmail = async ( clientRequestId: string, userId: string ): Promise => { - const user = await userRepository.findOneBy({ - id: userId, - }) + const user = await userRepository.findById(userId) if (!user) { return false } diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index 39babc270..870d9ce6b 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -1,4 +1,4 @@ -import { User } from '../entity/user' +import { StatusType, User } from '../entity/user' import { authTrx } from '../repository' import { userRepository } from '../repository/user' @@ -21,5 +21,5 @@ export const updateUser = async (userId: string, update: Partial) => { } export const findUser = async (id: string): Promise => { - return userRepository.findOneBy({ id }) + return userRepository.findOneBy({ id, status: StatusType.Active }) } diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index f8ca6ffa2..af7e40a9a 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -16,6 +16,7 @@ import { ILike } from 'typeorm' import { promisify } from 'util' import { v4 as uuid } from 'uuid' import { Highlight } from '../entity/highlight' +import { StatusType } from '../entity/user' import { env } from '../env' import { PageType, PreparedDocumentInput } from '../generated/graphql' import { userRepository } from '../repository/user' @@ -470,6 +471,7 @@ export const isProbablyArticle = async ( ): Promise => { const user = await userRepository.findOneBy({ email: ILike(email), + status: StatusType.Active, }) return !!user || subject.includes(ARTICLE_PREFIX) } diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 07393fbdb..7a9db963d 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -146,7 +146,7 @@ describe('Subscriptions API', () => { undefined, SubscriptionType.Newsletter ) - const allSubscriptions = [sub5, ...subscriptions] + const allSubscriptions = [...subscriptions, sub5] const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.subscriptions.subscriptions).to.eql( diff --git a/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql new file mode 100755 index 000000000..d8b83487d --- /dev/null +++ b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql @@ -0,0 +1,19 @@ +-- Type: DO +-- Name: add_unique_to_subscription_url +-- Description: Add unique constraint to the url field on the omnivore.subscription table + +BEGIN; + +-- Deleting duplicates first to avoid unique constraint violation +WITH DuplicateCTE AS ( + SELECT user_id, url, + ROW_NUMBER() OVER (PARTITION BY user_id, url ORDER BY (SELECT NULL)) AS RowNum + FROM omnivore.subscriptions + WHERE type = 'RSS' +) +DELETE FROM omnivore.subscriptions + WHERE (user_id, url) IN (SELECT user_id, url FROM DuplicateCTE WHERE RowNum > 1); + +ALTER TABLE omnivore.subscriptions ADD CONSTRAINT subscriptions_user_id_url_key UNIQUE (user_id, url); + +COMMIT; diff --git a/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql b/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql new file mode 100755 index 000000000..72071eab2 --- /dev/null +++ b/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_unique_to_subscription_url +-- Description: Add unique constraint to the url field on the omnivore.subscription table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP CONSTRAINT IF EXISTS subscriptions_user_id_url_key; + +COMMIT; From a537252db3f482c7b96ca1409538f101d971ba36 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 17:50:04 +0800 Subject: [PATCH 12/78] Limit CSV import sizes via web --- packages/web/components/templates/UploadModal.tsx | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/web/components/templates/UploadModal.tsx b/packages/web/components/templates/UploadModal.tsx index 8c0422f94..ec8b296c1 100644 --- a/packages/web/components/templates/UploadModal.tsx +++ b/packages/web/components/templates/UploadModal.tsx @@ -122,7 +122,7 @@ export function UploadModal(props: UploadModalProps): JSX.Element { const uploadSignedUrlForFile = async ( file: UploadingFile ): Promise => { - let { contentType } = file; + let { contentType } = file if ( contentType == 'application/vnd.ms-excel' && file.name.endsWith('.csv') @@ -135,6 +135,12 @@ export function UploadModal(props: UploadModalProps): JSX.Element { try { const csvData = await validateCsvFile(file.file) urlCount = csvData.data.length + if (urlCount > 500) { + return { + message: + 'Due to an increase in traffic we are limiting CSV imports to 500 items.', + } + } if (csvData.inValidData.length > 0) { return { message: csvData.inValidData[0].message, @@ -185,7 +191,7 @@ export function UploadModal(props: UploadModalProps): JSX.Element { } } return { - message: `Invalid content type: ${contentType}` + message: `Invalid content type: ${contentType}`, } } @@ -215,8 +221,9 @@ export function UploadModal(props: UploadModalProps): JSX.Element { const uploadInfo = await uploadSignedUrlForFile(file) if (!uploadInfo.uploadSignedUrl) { const message = uploadInfo.message || 'No upload URL available' - // close after 5 seconds - showErrorToast(message, { duration: 5000 }) + showErrorToast(message, { duration: 10000 }) + file.status = 'error' + setUploadFiles([...allFiles]) return } From 74b25a23ac29ffd4a2da0c0ff20f6e6b06f9db6d Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 17:51:28 +0800 Subject: [PATCH 13/78] Temporarily remove next/previous web controls --- .../web/pages/[username]/[slug]/index.tsx | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/packages/web/pages/[username]/[slug]/index.tsx b/packages/web/pages/[username]/[slug]/index.tsx index 2acd23610..f6ab12ece 100644 --- a/packages/web/pages/[username]/[slug]/index.tsx +++ b/packages/web/pages/[username]/[slug]/index.tsx @@ -79,30 +79,30 @@ export default function Home(): JSX.Element { }, [articleData?.article.article]) const goNextOrHome = useCallback(() => { - const listStr = localStorage.getItem('library-slug-list') - if (article && listStr && viewerData?.me) { - const libraryList = JSON.parse(listStr) as string[] - const idx = libraryList.findIndex((slug) => slug == article.slug) - if (idx != -1 && idx < libraryList.length - 1) { - const nextSlug = libraryList[idx + 1] as string - router.push(`/${viewerData?.me.profile.username}/${nextSlug}`) - return - } - } + // const listStr = localStorage.getItem('library-slug-list') + // if (article && listStr && viewerData?.me) { + // const libraryList = JSON.parse(listStr) as string[] + // const idx = libraryList.findIndex((slug) => slug == article.slug) + // if (idx != -1 && idx < libraryList.length - 1) { + // const nextSlug = libraryList[idx + 1] as string + // router.push(`/${viewerData?.me.profile.username}/${nextSlug}`) + // return + // } + // } router.push(`/home`) }, [router, viewerData, article]) const goPreviousOrHome = useCallback(() => { - const listStr = localStorage.getItem('library-slug-list') - if (article && listStr && viewerData?.me) { - const libraryList = JSON.parse(listStr) as string[] - const idx = libraryList.findIndex((slug) => slug == article.slug) - if (idx > 0) { - const previousSlug = libraryList[idx - 1] as string - router.push(`/${viewerData?.me.profile.username}/${previousSlug}`) - return - } - } + // const listStr = localStorage.getItem('library-slug-list') + // if (article && listStr && viewerData?.me) { + // const libraryList = JSON.parse(listStr) as string[] + // const idx = libraryList.findIndex((slug) => slug == article.slug) + // if (idx > 0) { + // const previousSlug = libraryList[idx - 1] as string + // router.push(`/${viewerData?.me.profile.username}/${previousSlug}`) + // return + // } + // } router.push(`/home`) }, [router, viewerData, article]) @@ -418,24 +418,24 @@ export default function Home(): JSX.Element { shortcut: ['i'], perform: () => setShowEditModal(true), }, - { - id: 'go_previous', - section: 'Article', - name: 'Go to Previous', - shortcut: ['g', 'p'], - perform: () => { - document.dispatchEvent(new Event('goPreviousOrHome')) - }, - }, - { - id: 'go_next', - section: 'Article', - name: 'Go to Next', - shortcut: ['g', 'n'], - perform: () => { - document.dispatchEvent(new Event('goNextOrHome')) - }, - }, + // { + // id: 'go_previous', + // section: 'Article', + // name: 'Go to Previous', + // shortcut: ['g', 'p'], + // perform: () => { + // document.dispatchEvent(new Event('goPreviousOrHome')) + // }, + // }, + // { + // id: 'go_next', + // section: 'Article', + // name: 'Go to Next', + // shortcut: ['g', 'n'], + // perform: () => { + // document.dispatchEvent(new Event('goNextOrHome')) + // }, + // }, ], [readerSettings, showHighlightsModal] ) From e4332b74f97b88eda9c702fa34c0f699da679725 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 20:11:51 +0800 Subject: [PATCH 14/78] Add queue manager service --- packages/queue-manager/Dockerfile | 26 +++ packages/queue-manager/mocha-config.json | 5 + packages/queue-manager/package.json | 31 +++ packages/queue-manager/src/index.ts | 190 ++++++++++++++++++ packages/queue-manager/test/babel-register.js | 3 + packages/queue-manager/test/stub.test.ts | 8 + packages/queue-manager/tsconfig.json | 8 + 7 files changed, 271 insertions(+) create mode 100644 packages/queue-manager/Dockerfile create mode 100644 packages/queue-manager/mocha-config.json create mode 100644 packages/queue-manager/package.json create mode 100644 packages/queue-manager/src/index.ts create mode 100644 packages/queue-manager/test/babel-register.js create mode 100644 packages/queue-manager/test/stub.test.ts create mode 100644 packages/queue-manager/tsconfig.json diff --git a/packages/queue-manager/Dockerfile b/packages/queue-manager/Dockerfile new file mode 100644 index 000000000..fe547f483 --- /dev/null +++ b/packages/queue-manager/Dockerfile @@ -0,0 +1,26 @@ +FROM node:14.18-alpine + +# Run everything after as non-privileged user. +WORKDIR /app + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .eslintrc . + +COPY /packages/queue-manager/package.json ./packages/queue-manager/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/queue-manager ./packages/queue-manager +RUN yarn workspace @omnivore/queue-manager build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/queue-manager/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 8080 + +CMD ["yarn", "workspace", "@omnivore/queue-manager", "start"] + diff --git a/packages/queue-manager/mocha-config.json b/packages/queue-manager/mocha-config.json new file mode 100644 index 000000000..44d1d24c1 --- /dev/null +++ b/packages/queue-manager/mocha-config.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "spec": "test/**/*.test.ts", + "require": "test/babel-register.js" + } \ No newline at end of file diff --git a/packages/queue-manager/package.json b/packages/queue-manager/package.json new file mode 100644 index 000000000..69c2630a8 --- /dev/null +++ b/packages/queue-manager/package.json @@ -0,0 +1,31 @@ +{ + "name": "@omnivore/queue-manager", + "version": "1.0.0", + "main": "build/src/index.js", + "files": [ + "build/src" + ], + "license": "Apache-2.0", + "scripts": { + "test": "yarn mocha -r ts-node/register --config mocha-config.json", + "lint": "eslint src --ext ts,js,tsx,jsx", + "compile": "tsc", + "build": "tsc", + "start": "functions-framework --target=queueManager", + "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" + }, + "devDependencies": { + "chai": "^4.3.6", + "eslint-plugin-prettier": "^4.0.0", + "mocha": "^10.0.0" + }, + "dependencies": { + "@google-cloud/functions-framework": "3.1.2", + "@google-cloud/monitoring": "^4.0.0", + "@google-cloud/tasks": "^4.0.0", + "@sentry/serverless": "^6.16.1", + "axios": "^1.4.0", + "dotenv": "^16.0.1", + "jsonwebtoken": "^8.5.1" + } +} \ No newline at end of file diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts new file mode 100644 index 000000000..f2b5791c5 --- /dev/null +++ b/packages/queue-manager/src/index.ts @@ -0,0 +1,190 @@ +import { MetricServiceClient } from '@google-cloud/monitoring' +import { v2beta3 } from '@google-cloud/tasks' +import fetch from 'node-fetch' +import * as dotenv from 'dotenv' + +import * as Sentry from '@sentry/serverless' + +dotenv.config() +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) + +const PROJECT_ID = process.env.PROJECT_ID +const LOCATION = 'us-west2' +const IMPORT_QUEUE_NAME = process.env.IMPORT_QUEUE_NAME +const RSS_QUEUE_NAME = process.env.RSS_FEED_QUEUE_NAME +const QUEUE_NAMES = [IMPORT_QUEUE_NAME, RSS_QUEUE_NAME] +const DISCORD_WEBHOOK_URL = process.env.DISCORD_WEBHOOK_URL +const METRICS_FILTER = `metric.type="appengine.googleapis.com/http/server/response_latencies" metric.labels.response_code="200"` + +if ( + !PROJECT_ID || + !IMPORT_QUEUE_NAME || + !RSS_QUEUE_NAME || + !DISCORD_WEBHOOK_URL +) { + throw new Error('environment not supplied.') +} + +const LATENCY_THRESHOLD = 500 +const RSS_QUEUE_THRESHOLD = 20_000 +const IMPORT_QUEUE_THRESHOLD = 100_000 + +const postToDiscord = async (message: string) => { + console.log('notify message', { message }) + const payload = { + content: message, + } + + try { + const response = await fetch(DISCORD_WEBHOOK_URL, { + method: 'POST', + body: JSON.stringify(payload), + headers: { + 'Content-Type': 'application/json', + }, + }) + + if (!response.ok) { + throw new Error(`Discord response was not ok: ${response.statusText}`) + } + } catch (error) { + console.error('Failed to post message to Discord:', error) + } +} + +const checkShouldPauseQueues = async () => { + const now = Date.now() + const client = new MetricServiceClient() + + // Query for the metrics from the last 5 minutes + const [timeSeries] = await client.listTimeSeries({ + name: client.projectPath(PROJECT_ID), + filter: METRICS_FILTER, + interval: { + startTime: { + seconds: Math.floor(now / 1000 - 5 * 60), + }, + endTime: { + seconds: Math.floor(now / 1000), + }, + }, + aggregation: { + alignmentPeriod: { + seconds: 300, + }, + perSeriesAligner: 'ALIGN_PERCENTILE_95', + }, + }) + + let shouldPauseQueues = false + + for (const ts of timeSeries) { + // We only want to look at the backend service right now + if ( + !ts.resource || + !ts.resource.labels || + !ts.resource.labels['module_id'] || + !ts.resource.labels['module_id'].startsWith('backend') + ) { + continue + } + + if (ts.points && ts.points.length) { + const avgLatency = + ts.points.reduce( + (acc, point) => acc + (point.value?.doubleValue ?? 0), + 0 + ) / ts.points.length + console.log('avgLatency: ', avgLatency) + if (avgLatency > LATENCY_THRESHOLD) { + shouldPauseQueues = true + break + } + } + } + + return shouldPauseQueues +} + +const getQueueTaskCount = async (queueName: string) => { + const cloudTasksClient = new v2beta3.CloudTasksClient() + const queuePath = cloudTasksClient.queuePath(PROJECT_ID, LOCATION, queueName) + const [queue] = await cloudTasksClient.getQueue({ + name: queuePath, + readMask: { paths: ['name', 'stats'] }, + }) + + console.log(' queue.stats', { stats: queue.stats }) + if (Number.isNaN(queue.stats?.tasksCount)) { + return 0 + } + return Number(queue.stats?.tasksCount) +} + +const pauseQueues = async () => { + const cloudTasksClient = new v2beta3.CloudTasksClient() + + await Promise.all([ + cloudTasksClient.pauseQueue({ + name: cloudTasksClient.queuePath(PROJECT_ID, LOCATION, RSS_QUEUE_NAME), + }), + cloudTasksClient.pauseQueue({ + name: cloudTasksClient.queuePath(PROJECT_ID, LOCATION, IMPORT_QUEUE_NAME), + }), + ]) +} + +async function checkMetricsAndPauseQueues() { + if ( + !PROJECT_ID || + !IMPORT_QUEUE_NAME || + !RSS_QUEUE_NAME || + !DISCORD_WEBHOOK_URL + ) { + throw new Error('environment not supplied.') + } + + const shouldPauseQueues = await checkShouldPauseQueues() + + if (shouldPauseQueues) { + const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` + + // Pause the two queues + await pauseQueues() + + // Post to Discord server using webhook + await postToDiscord(message) + } else { + const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + + if (rssQueueCount > RSS_QUEUE_THRESHOLD) { + await postToDiscord( + `The RSS queue has exceeded it's threshold, it has ${rssQueueCount} items in it.` + ) + } + + if (importQueueCount > IMPORT_QUEUE_THRESHOLD) { + await postToDiscord( + `The import queue has exceeded it's threshold, it has ${importQueueCount} items in it.` + ) + } + } +} + +export const queueManager = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + try { + checkMetricsAndPauseQueues() + res.send('ok') + } catch (e) { + console.error('Error while parsing RSS feed', e) + res.status(500).send('INTERNAL_SERVER_ERROR') + } + } +) diff --git a/packages/queue-manager/test/babel-register.js b/packages/queue-manager/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/queue-manager/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/queue-manager/test/stub.test.ts b/packages/queue-manager/test/stub.test.ts new file mode 100644 index 000000000..24ad25c8f --- /dev/null +++ b/packages/queue-manager/test/stub.test.ts @@ -0,0 +1,8 @@ +import 'mocha' +import { expect } from 'chai' + +describe('stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/queue-manager/tsconfig.json b/packages/queue-manager/tsconfig.json new file mode 100644 index 000000000..7ebe093f6 --- /dev/null +++ b/packages/queue-manager/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./../../tsconfig.json", + "compilerOptions": { + "outDir": "build", + "rootDir": "." + }, + "include": ["src"] +} From 51e2aaa3eae6134d473a72af2c5e801bcacc8807 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 20:29:03 +0800 Subject: [PATCH 15/78] Add node-fetch dependency --- packages/queue-manager/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/queue-manager/package.json b/packages/queue-manager/package.json index 69c2630a8..7c1319a45 100644 --- a/packages/queue-manager/package.json +++ b/packages/queue-manager/package.json @@ -15,6 +15,7 @@ "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" }, "devDependencies": { + "@types/node-fetch": "^2.6.6", "chai": "^4.3.6", "eslint-plugin-prettier": "^4.0.0", "mocha": "^10.0.0" @@ -28,4 +29,4 @@ "dotenv": "^16.0.1", "jsonwebtoken": "^8.5.1" } -} \ No newline at end of file +} From b700c75552f6382537b022f93e071874249f9de2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 22:32:41 +0800 Subject: [PATCH 16/78] Use the GCP project id --- packages/queue-manager/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index f2b5791c5..1e79a304d 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -11,7 +11,7 @@ Sentry.GCPFunction.init({ tracesSampleRate: 0, }) -const PROJECT_ID = process.env.PROJECT_ID +const PROJECT_ID = process.env.GCP_PROJECT_ID const LOCATION = 'us-west2' const IMPORT_QUEUE_NAME = process.env.IMPORT_QUEUE_NAME const RSS_QUEUE_NAME = process.env.RSS_FEED_QUEUE_NAME From ab0fc1087d5cd2429d476eed61793126ab16eeda Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 12:02:44 +0800 Subject: [PATCH 17/78] update tsv only if necessary --- ...7.do.alter_library_item_tsv_update_trigger.sql | 15 +++++++++++++++ ...undo.alter_library_item_tsv_update_trigger.sql | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100755 packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql create mode 100755 packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql diff --git a/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql b/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql new file mode 100755 index 000000000..aa7d266d8 --- /dev/null +++ b/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql @@ -0,0 +1,15 @@ +-- Type: DO +-- Name: alter_library_item_tsv_update_trigger +-- Description: Alter library_item_tsv_update trigger on omnivore.library_item table to add conditions + +BEGIN; + +DROP TRIGGER IF EXISTS library_item_tsv_update ON omnivore.library_item; + +CREATE TRIGGER library_item_tsv_update + BEFORE INSERT OR UPDATE OF readable_content, site_name, title, author, description, note, highlight_annotations + ON omnivore.library_item + FOR EACH ROW + EXECUTE PROCEDURE update_library_item_tsv(); + +COMMIT; diff --git a/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql b/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql new file mode 100755 index 000000000..cb367e944 --- /dev/null +++ b/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql @@ -0,0 +1,15 @@ +-- Type: UNDO +-- Name: alter_library_item_tsv_update_trigger +-- Description: Alter library_item_tsv_update trigger on omnivore.library_item table to add conditions + +BEGIN; + +DROP TRIGGER IF EXISTS library_item_tsv_update ON omnivore.library_item; + +CREATE TRIGGER library_item_tsv_update + BEFORE INSERT OR UPDATE + ON omnivore.library_item + FOR EACH ROW + EXECUTE PROCEDURE update_library_item_tsv(); + +COMMIT; From 1f600c272e7208c11822bba61118ad2fbe4d86fe Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 13:36:47 +0800 Subject: [PATCH 18/78] Better handling of network failures for labels/subscriptions/searches in the left menu --- .../templates/homeFeed/LibraryFilterMenu.tsx | 41 +++++++++++-------- .../networking/queries/useGetLabelsQuery.tsx | 5 ++- .../queries/useGetSavedSearchQuery.tsx | 3 ++ .../queries/useGetSubscriptionsQuery.tsx | 3 ++ 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx index 11adec862..fba2015c9 100644 --- a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx +++ b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx @@ -48,32 +48,39 @@ export function LibraryFilterMenu(props: LibraryFilterMenuProps): JSX.Element { isSessionStorage: false, initialValue: [], }) - const { labels: networkLabels, isLoading: labelsLoading } = - useGetLabelsQuery() - const { savedSearches: networkSearches, isLoading: searchesLoading } = - useGetSavedSearchQuery() - const { - subscriptions: networkSubscriptions, - isLoading: subscriptionsLoading, - } = useGetSubscriptionsQuery() + const labelsResponse = useGetLabelsQuery() + const searchesResponse = useGetSavedSearchQuery() + const subscriptionsResponse = useGetSubscriptionsQuery() useEffect(() => { - if (!labelsLoading) { - setLabels(networkLabels) + if ( + !labelsResponse.error && + !labelsResponse.isLoading && + labelsResponse.labels + ) { + setLabels(labelsResponse.labels) } - }, [setLabels, networkLabels, labelsLoading]) + }, [setLabels, labelsResponse]) useEffect(() => { - if (!subscriptionsLoading) { - setSubscriptions(networkSubscriptions) + if ( + !subscriptionsResponse.error && + !subscriptionsResponse.isLoading && + subscriptionsResponse.subscriptions + ) { + setSubscriptions(subscriptionsResponse.subscriptions) } - }, [setSubscriptions, networkSubscriptions, subscriptionsLoading]) + }, [setSubscriptions, subscriptionsResponse]) useEffect(() => { - if (!searchesLoading) { - setSavedSearches(networkSearches ?? []) + if ( + !searchesResponse.error && + !searchesResponse.isLoading && + searchesResponse.savedSearches + ) { + setSavedSearches(searchesResponse.savedSearches) } - }, [setSavedSearches, networkSearches, searchesLoading]) + }, [setSavedSearches, searchesResponse]) return ( <> diff --git a/packages/web/lib/networking/queries/useGetLabelsQuery.tsx b/packages/web/lib/networking/queries/useGetLabelsQuery.tsx index d6fefad8e..abd4295fd 100644 --- a/packages/web/lib/networking/queries/useGetLabelsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetLabelsQuery.tsx @@ -4,6 +4,7 @@ import { Label, labelFragment } from '../fragments/labelFragment' import { publicGqlFetcher } from '../networkHelpers' type LabelsQueryResponse = { + error: any isLoading: boolean isValidating: boolean labels: Label[] @@ -38,10 +39,11 @@ export function useGetLabelsQuery(): LabelsQueryResponse { const { data, error, mutate, isValidating } = useSWR(query, publicGqlFetcher) try { - if (data) { + if (data && !error) { const result = data as LabelsResponseData const labels = result.labels?.labels as Label[] return { + error, isLoading: !error && !data, isValidating, labels, @@ -54,6 +56,7 @@ export function useGetLabelsQuery(): LabelsQueryResponse { console.log('error', error) } return { + error, isLoading: !error && !data, isValidating: false, labels: [], diff --git a/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx b/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx index 4dde76f89..f6a76f386 100644 --- a/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx +++ b/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx @@ -7,6 +7,7 @@ import { } from '../fragments/savedSearchFragment' type SavedSearchResponse = { + error: any savedSearches?: SavedSearch[] savedSearchErrors?: unknown isLoading: boolean @@ -39,6 +40,7 @@ export function useGetSavedSearchQuery(): SavedSearchResponse { const { filters } = data as SavedSearchResponseData return { + error, savedSearches: filters?.filters ?? [], savedSearchErrors: error ?? {}, isLoading: false, @@ -46,6 +48,7 @@ export function useGetSavedSearchQuery(): SavedSearchResponse { } return { + error, savedSearches: [], savedSearchErrors: null, isLoading: !error && !data, diff --git a/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx b/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx index 4c0a26d17..ababa6fdb 100644 --- a/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx @@ -25,6 +25,7 @@ export type Subscription = { } type SubscriptionsQueryResponse = { + error: any isLoading: boolean isValidating: boolean subscriptions: Subscription[] @@ -85,6 +86,7 @@ export function useGetSubscriptionsQuery( const result = data as SubscriptionsResponseData const subscriptions = result.subscriptions.subscriptions as Subscription[] return { + error, isLoading: !error && !data, isValidating, subscriptions, @@ -97,6 +99,7 @@ export function useGetSubscriptionsQuery( console.log('error', error) } return { + error, isLoading: !error && !data, isValidating: true, subscriptions: [], From ffa476a109ee44726596125603bd4355b25d9cba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Oct 2023 05:44:10 +0000 Subject: [PATCH 19/78] Bump @babel/traverse from 7.14.7 to 7.23.2 in /pkg/admin Bumps [@babel/traverse](https://github.com/babel/babel/tree/HEAD/packages/babel-traverse) from 7.14.7 to 7.23.2. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.23.2/packages/babel-traverse) --- updated-dependencies: - dependency-name: "@babel/traverse" dependency-type: indirect ... Signed-off-by: dependabot[bot] --- pkg/admin/yarn.lock | 144 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 132 insertions(+), 12 deletions(-) diff --git a/pkg/admin/yarn.lock b/pkg/admin/yarn.lock index eb2623a42..0b67d4ca9 100644 --- a/pkg/admin/yarn.lock +++ b/pkg/admin/yarn.lock @@ -33,6 +33,14 @@ dependencies: "@babel/highlight" "^7.14.5" +"@babel/code-frame@^7.22.13": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== + dependencies: + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" + "@babel/compat-data@^7.13.11", "@babel/compat-data@^7.14.5", "@babel/compat-data@^7.14.7": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.7.tgz#7b047d7a3a89a67d2258dc61f604f098f1bc7e08" @@ -68,6 +76,16 @@ jsesc "^2.5.1" source-map "^0.5.0" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.0.0", "@babel/helper-annotate-as-pure@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.14.5.tgz#7bf478ec3b71726d56a8ca5775b046fc29879e61" @@ -127,6 +145,11 @@ resolve "^1.14.2" semver "^6.1.2" +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + "@babel/helper-explode-assignable-expression@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.14.5.tgz#8aa72e708205c7bb643e45c73b4386cdf2a1f645" @@ -143,6 +166,14 @@ "@babel/template" "^7.14.5" "@babel/types" "^7.14.5" +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== + dependencies: + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" + "@babel/helper-get-function-arity@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.14.5.tgz#25fbfa579b0937eee1f3b805ece4ce398c431815" @@ -157,6 +188,13 @@ dependencies: "@babel/types" "^7.14.5" +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-member-expression-to-functions@^7.14.5": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.14.7.tgz#97e56244beb94211fe277bd818e3a329c66f7970" @@ -237,11 +275,28 @@ dependencies: "@babel/types" "^7.14.5" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + +"@babel/helper-string-parser@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" + integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== + "@babel/helper-validator-identifier@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz#d0f0e277c512e0c938277faa85a3968c9a44c0e8" integrity sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-option@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.14.5.tgz#6e72a1fff18d5dfcb878e1e62f1a021c4b72d5a3" @@ -275,11 +330,25 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.10.2", "@babel/parser@^7.12.5", "@babel/parser@^7.14.5", "@babel/parser@^7.14.6", "@babel/parser@^7.14.7": +"@babel/highlight@^7.22.13": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" + js-tokens "^4.0.0" + +"@babel/parser@^7.10.2", "@babel/parser@^7.12.5", "@babel/parser@^7.14.5", "@babel/parser@^7.14.6": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.7.tgz#6099720c8839ca865a2637e6c85852ead0bdb595" integrity sha512-X67Z5y+VBJuHB/RjwECp8kSl5uYi0BvRbNeWqkaJCVh+LiTPl19WBUfG627psSgp9rSf6ojuXghQM3ha6qHHdA== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.14.5.tgz#4b467302e1548ed3b1be43beae2cc9cf45e0bb7e" @@ -977,18 +1046,28 @@ "@babel/parser" "^7.14.5" "@babel/types" "^7.14.5" -"@babel/traverse@^7.12.5", "@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.4.5": - version "7.14.7" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.7.tgz#64007c9774cfdc3abd23b0780bc18a3ce3631753" - integrity sha512-9vDr5NzHu27wgwejuKL7kIOm4bwEtaPQ4Z6cpCmjSuaRqpH/7xc4qcGEscwMqlkwgcXl6MvqoAjZkQ24uSdIZQ== +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== dependencies: - "@babel/code-frame" "^7.14.5" - "@babel/generator" "^7.14.5" - "@babel/helper-function-name" "^7.14.5" - "@babel/helper-hoist-variables" "^7.14.5" - "@babel/helper-split-export-declaration" "^7.14.5" - "@babel/parser" "^7.14.7" - "@babel/types" "^7.14.5" + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.12.5", "@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.4.5": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -1000,6 +1079,15 @@ "@babel/helper-validator-identifier" "^7.14.5" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@carbon/icon-helpers@^10.19.0": version "10.19.0" resolved "https://registry.yarnpkg.com/@carbon/icon-helpers/-/icon-helpers-10.19.0.tgz#f6b608b181b4ca4aeeadac72ec11b7cf530b4d1c" @@ -1131,6 +1219,38 @@ gud "^1.0.0" warning "^4.0.3" +"@jridgewell/gen-mapping@^0.3.2": + version "0.3.3" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098" + integrity sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ== + dependencies: + "@jridgewell/set-array" "^1.0.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.9" + +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" + integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + +"@jridgewell/set-array@^1.0.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" + integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== + +"@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9": + version "0.3.19" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz#f8a3249862f91be48d3127c3cfe992f79b4b8811" + integrity sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" From d141fea9777838299a012ce95eb34cc6dea6c680 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 14:18:45 +0800 Subject: [PATCH 20/78] Handle cases where fetching queue counts fails --- packages/queue-manager/src/index.ts | 50 +++++++++++++++++------------ 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 1e79a304d..190903a96 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -150,29 +150,37 @@ async function checkMetricsAndPauseQueues() { const shouldPauseQueues = await checkShouldPauseQueues() if (shouldPauseQueues) { - const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) - const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) - const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` - - // Pause the two queues - await pauseQueues() - - // Post to Discord server using webhook - await postToDiscord(message) - } else { - const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) - const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) - - if (rssQueueCount > RSS_QUEUE_THRESHOLD) { - await postToDiscord( - `The RSS queue has exceeded it's threshold, it has ${rssQueueCount} items in it.` - ) + let rssQueueCount: number | string = 'unknown' + let importQueueCount: number | string = 'unknown' + try { + rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + } catch (err) { + console.log('error fetching queue counts', err) } - if (importQueueCount > IMPORT_QUEUE_THRESHOLD) { - await postToDiscord( - `The import queue has exceeded it's threshold, it has ${importQueueCount} items in it.` - ) + const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` + + await pauseQueues() + await postToDiscord(message) + } else { + try { + const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + + if (rssQueueCount > RSS_QUEUE_THRESHOLD) { + await postToDiscord( + `The RSS queue has exceeded it's threshold, it has ${rssQueueCount} items in it.` + ) + } + + if (importQueueCount > IMPORT_QUEUE_THRESHOLD) { + await postToDiscord( + `The import queue has exceeded it's threshold, it has ${importQueueCount} items in it.` + ) + } + } catch (err) { + console.log('error getting queue counts') } } } From a6d3bb22b11e192d50aa7a87d8cf9e5ade3b4811 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 14:46:57 +0800 Subject: [PATCH 21/78] Add a check param so we can avoid checks on container startup while deploying --- packages/queue-manager/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 190903a96..4d560b337 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -188,7 +188,9 @@ async function checkMetricsAndPauseQueues() { export const queueManager = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { try { - checkMetricsAndPauseQueues() + if (req.query['check']) { + checkMetricsAndPauseQueues() + } res.send('ok') } catch (e) { console.error('Error while parsing RSS feed', e) From eb667e04b92246d0819937140fa581f27741e1e8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 15:09:16 +0800 Subject: [PATCH 22/78] update reading progress in db --- packages/api/src/resolvers/article/index.ts | 62 +++++++----------- packages/api/src/services/library_item.ts | 69 +++++++++++++++++++++ packages/api/test/resolvers/article.test.ts | 1 - 3 files changed, 90 insertions(+), 42 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index bf26872c5..bf7dc89f9 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -62,11 +62,11 @@ import { } from '../../services/labels' import { createLibraryItem, - findLibraryItemById, findLibraryItemByUrl, findLibraryItemsByPrefix, searchLibraryItems, updateLibraryItem, + updateLibraryItemReadingProgress, updateLibraryItems, } from '../../services/library_item' import { parsedContentToLibraryItem } from '../../services/save_page' @@ -572,14 +572,8 @@ export const saveArticleReadingProgressResolver = authorized< readingProgressTopPercent, }, }, - { uid, pubsub } + { log, pubsub, uid } ) => { - const libraryItem = await findLibraryItemById(id, uid) - - if (!libraryItem) { - return { errorCodes: [SaveArticleReadingProgressErrorCode.NotFound] } - } - if ( readingProgressPercent < 0 || readingProgressPercent > 100 || @@ -590,40 +584,26 @@ export const saveArticleReadingProgressResolver = authorized< ) { return { errorCodes: [SaveArticleReadingProgressErrorCode.BadData] } } - // If we have a top percent, we only save it if it's greater than the current top percent - // or set to zero if the top percent is zero. - const readingProgressTopPercentToSave = readingProgressTopPercent - ? Math.max( - readingProgressTopPercent, - libraryItem.readingProgressTopPercent || 0 - ) - : readingProgressTopPercent === 0 - ? 0 - : undefined - // If setting to zero we accept the update, otherwise we require it - // be greater than the current reading progress. - const updatedPart: QueryDeepPartialEntity = { - readingProgressBottomPercent: - readingProgressPercent === 0 - ? 0 - : Math.max( - readingProgressPercent, - libraryItem.readingProgressBottomPercent - ), - readingProgressHighestReadAnchor: - readingProgressAnchorIndex === 0 - ? 0 - : Math.max( - readingProgressAnchorIndex || 0, - libraryItem.readingProgressHighestReadAnchor - ), - readingProgressTopPercent: readingProgressTopPercentToSave, - readAt: new Date(), - } - const updatedItem = await updateLibraryItem(id, updatedPart, uid, pubsub) + try { + const updatedItem = await updateLibraryItemReadingProgress( + id, + uid, + readingProgressPercent, + readingProgressTopPercent, + readingProgressAnchorIndex, + pubsub + ) + if (!updatedItem) { + return { errorCodes: [SaveArticleReadingProgressErrorCode.BadData] } + } - return { - updatedArticle: libraryItemToArticle(updatedItem), + return { + updatedArticle: libraryItemToArticle(updatedItem), + } + } catch (error) { + log.error('saveArticleReadingProgressResolver error', error) + + return { errorCodes: [SaveArticleReadingProgressErrorCode.Unauthorized] } } } ) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 07be9f885..8dab74fc5 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -438,6 +438,75 @@ export const updateLibraryItem = async ( return updatedLibraryItem } +export const updateLibraryItemReadingProgress = async ( + id: string, + userId: string, + bottomPercent: number, + topPercent: number | null = null, + anchorIndex: number | null = null, + pubsub = createPubSubClient() +): Promise => { + // If we have a top percent, we only save it if it's greater than the current top percent + // or set to zero if the top percent is zero. + const result = (await authTrx( + async (tx) => + tx.getRepository(LibraryItem).query( + ` + UPDATE omnivore.library_item + SET reading_progress_top_percent = CASE + WHEN reading_progress_top_percent < $2 THEN $2 + WHEN $2 = 0 THEN 0 + ELSE reading_progress_top_percent + END, + reading_progress_bottom_percent = CASE + WHEN reading_progress_bottom_percent < $3 THEN $3 + WHEN $3 = 0 THEN 0 + ELSE reading_progress_bottom_percent + END, + reading_progress_highest_read_anchor = CASE + WHEN reading_progress_top_percent < $4 THEN $4 + WHEN $4 = 0 THEN 0 + ELSE reading_progress_highest_read_anchor + END, + read_at = now() + WHERE id = $1 AND ( + (reading_progress_top_percent < $2 OR $2 = 0) OR + (reading_progress_bottom_percent < $3 OR $3 = 0) OR + (reading_progress_highest_read_anchor < $4 OR $4 = 0) + ) + RETURNING + id, + reading_progress_top_percent as "readingProgressTopPercent", + reading_progress_bottom_percent as "readingProgressBottomPercent", + reading_progress_highest_read_anchor as "readingProgressHighestReadAnchor", + read_at as "readAt" + `, + [id, topPercent, bottomPercent, anchorIndex] + ), + undefined, + userId + )) as [LibraryItem[], number] + if (result[1] === 0) { + return null + } + + const updatedItem = result[0][0] + await pubsub.entityUpdated>( + EntityType.PAGE, + { + id, + readingProgressBottomPercent: updatedItem.readingProgressBottomPercent, + readingProgressTopPercent: updatedItem.readingProgressTopPercent, + readingProgressHighestReadAnchor: + updatedItem.readingProgressHighestReadAnchor, + readAt: updatedItem.readAt, + }, + userId + ) + + return updatedItem +} + export const createLibraryItems = async ( libraryItems: DeepPartial[], userId: string diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index e3b13c0d1..083bd389e 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -706,7 +706,6 @@ describe('Article API', () => { ).to.eq(75) // Now try to set to a lower value (50), value should not be updated - // refresh index to ensure the reading progress is updated const secondQuery = saveArticleReadingProgressQuery(itemId, 50) const secondRes = await graphqlRequest(secondQuery, authToken).expect(200) expect( From 97cb8d3af7fb2694cd422f7002f409be7c316a21 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 15:53:05 +0800 Subject: [PATCH 23/78] Bump test limit --- packages/queue-manager/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 4d560b337..3ddc6a62e 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -30,7 +30,7 @@ if ( const LATENCY_THRESHOLD = 500 const RSS_QUEUE_THRESHOLD = 20_000 -const IMPORT_QUEUE_THRESHOLD = 100_000 +const IMPORT_QUEUE_THRESHOLD = 200_000 const postToDiscord = async (message: string) => { console.log('notify message', { message }) From d450bdd8e65f1c23faa9dcd30ea27f2fd2d24a8f Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 16:03:38 +0800 Subject: [PATCH 24/78] Linting improvements --- packages/queue-manager/.eslintignore | 2 ++ packages/queue-manager/.eslintrc | 6 ++++++ packages/queue-manager/src/index.ts | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 packages/queue-manager/.eslintignore create mode 100644 packages/queue-manager/.eslintrc diff --git a/packages/queue-manager/.eslintignore b/packages/queue-manager/.eslintignore new file mode 100644 index 000000000..b38db2f29 --- /dev/null +++ b/packages/queue-manager/.eslintignore @@ -0,0 +1,2 @@ +node_modules/ +build/ diff --git a/packages/queue-manager/.eslintrc b/packages/queue-manager/.eslintrc new file mode 100644 index 000000000..e006282a6 --- /dev/null +++ b/packages/queue-manager/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "tsconfig.json" + } +} \ No newline at end of file diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 3ddc6a62e..f26246914 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -189,7 +189,7 @@ export const queueManager = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { try { if (req.query['check']) { - checkMetricsAndPauseQueues() + await checkMetricsAndPauseQueues() } res.send('ok') } catch (e) { From 6e79f6c67fdb288fa1aa9dd73f6b0d73edd983be Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 16:41:14 +0800 Subject: [PATCH 25/78] update sql to deduplicate rss subscriptions --- packages/api/src/resolvers/subscriptions/index.ts | 1 - .../db/migrations/0136.do.add_unique_to_subscription_url.sql | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 7076e8524..74375441d 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -185,7 +185,6 @@ export const subscribeResolver = authorized< url: input.url || undefined, name: input.name || undefined, user: { id: uid }, - status: SubscriptionStatus.Active, type: input.subscriptionType || SubscriptionType.Rss, // default to rss }) ) diff --git a/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql index d8b83487d..2bfea4029 100755 --- a/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql +++ b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql @@ -6,13 +6,12 @@ BEGIN; -- Deleting duplicates first to avoid unique constraint violation WITH DuplicateCTE AS ( - SELECT user_id, url, - ROW_NUMBER() OVER (PARTITION BY user_id, url ORDER BY (SELECT NULL)) AS RowNum + SELECT id, ROW_NUMBER() OVER (PARTITION BY user_id, url ORDER BY status, last_fetched_at DESC NULLS LAST) AS row_number FROM omnivore.subscriptions WHERE type = 'RSS' ) DELETE FROM omnivore.subscriptions - WHERE (user_id, url) IN (SELECT user_id, url FROM DuplicateCTE WHERE RowNum > 1); + WHERE id IN (SELECT id FROM DuplicateCTE WHERE row_number > 1); ALTER TABLE omnivore.subscriptions ADD CONSTRAINT subscriptions_user_id_url_key UNIQUE (user_id, url); From 700fd73dbbd9894b44d8be3d7bbbfdd9edd7854f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 17:07:46 +0800 Subject: [PATCH 26/78] disallow social login if account is deleted --- packages/api/src/routers/auth/apple_auth.ts | 2 ++ packages/api/src/routers/auth/google_auth.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/api/src/routers/auth/apple_auth.ts b/packages/api/src/routers/auth/apple_auth.ts index 925e7d37e..99590ff4d 100644 --- a/packages/api/src/routers/auth/apple_auth.ts +++ b/packages/api/src/routers/auth/apple_auth.ts @@ -15,6 +15,7 @@ import { suggestedUsername, } from './jwt_helpers' import { analytics } from '../../utils/analytics' +import { StatusType } from '../../entity/user' const appleBaseURL = 'https://appleid.apple.com' const audienceName = 'app.omnivore.app' @@ -122,6 +123,7 @@ export async function handleAppleWebAuth( const user = await userRepository.findOneBy({ sourceUserId: decodedTokenResult.sourceUserId, source: 'APPLE', + status: StatusType.Active, }) const userId = user?.id diff --git a/packages/api/src/routers/auth/google_auth.ts b/packages/api/src/routers/auth/google_auth.ts index 781c45dcd..1108b6f23 100644 --- a/packages/api/src/routers/auth/google_auth.ts +++ b/packages/api/src/routers/auth/google_auth.ts @@ -1,6 +1,7 @@ import { google, oauth2_v2 as oauthV2 } from 'googleapis' import { OAuth2Client } from 'googleapis-common' import url from 'url' +import { StatusType } from '../../entity/user' import { env, homePageURL } from '../../env' import { LoginErrorCode } from '../../generated/graphql' import { userRepository } from '../../repository/user' @@ -130,6 +131,7 @@ export async function handleGoogleWebAuth( const user = await userRepository.findOneBy({ email, source: 'GOOGLE', + status: StatusType.Active, }) const userId = user?.id From 5f11bf1fded163844ff5ff1b84f7e464e2eae34b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 17:12:26 +0800 Subject: [PATCH 27/78] remove elasticsearch dep from packages/db --- packages/db/package.json | 1 - yarn.lock | 23 +---------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/packages/db/package.json b/packages/db/package.json index bf7cfd766..fce36c4af 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -9,7 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "@elastic/elasticsearch": "~7.12.0", "dotenv": "^8.2.0", "pg": "^8.3.0", "postgrator": "^4.1.1", diff --git a/yarn.lock b/yarn.lock index 3125e7f03..90f5c0f62 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2228,17 +2228,6 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw== -"@elastic/elasticsearch@~7.12.0": - version "7.12.0" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.12.0.tgz#dbb51a2841f644b670a56d8c15899e860928856f" - integrity sha512-GquUEytCijFRPEk3DKkkDdyhspB3qbucVQOwih9uNyz3iz804I+nGBUsFo2LwVvLQmQfEM0IY2+yoYfEz5wMug== - dependencies: - debug "^4.3.1" - hpagent "^0.1.1" - ms "^2.1.3" - pump "^3.0.0" - secure-json-parse "^2.3.1" - "@emotion/cache@^10.0.27": version "10.0.29" resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-10.0.29.tgz#87e7e64f412c060102d589fe7c6dc042e6f9d1e0" @@ -15835,11 +15824,6 @@ hpack.js@^2.1.6: readable-stream "^2.0.1" wbuf "^1.1.0" -hpagent@^0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9" - integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ== - html-encoding-sniffer@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" @@ -20177,7 +20161,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -24468,11 +24452,6 @@ search-query-parser@^1.6.0: resolved "https://registry.yarnpkg.com/search-query-parser/-/search-query-parser-1.6.0.tgz#d69ade33f3685cae25613a70189b7b18970b46f1" integrity sha512-bhf+phLlKF38nuniwLcVHWPArHGdzenlPhPi955CR3vm1QQifXIuPHwAffhjapojdVVzmv4hgIJ6NOX1d/w+Uw== -secure-json-parse@^2.3.1: - version "2.4.0" - resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85" - integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg== - selderee@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7" From 7eadda02e8f0735bea52849057095ba7d02aa34c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:18:58 +0800 Subject: [PATCH 28/78] Post the avg latency in the update message --- packages/queue-manager/src/index.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index f26246914..47383ba58 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -98,15 +98,13 @@ const checkShouldPauseQueues = async () => { (acc, point) => acc + (point.value?.doubleValue ?? 0), 0 ) / ts.points.length - console.log('avgLatency: ', avgLatency) if (avgLatency > LATENCY_THRESHOLD) { - shouldPauseQueues = true - break + return [true, avgLatency] } } } - return shouldPauseQueues + return [false, 0] } const getQueueTaskCount = async (queueName: string) => { @@ -147,7 +145,7 @@ async function checkMetricsAndPauseQueues() { throw new Error('environment not supplied.') } - const shouldPauseQueues = await checkShouldPauseQueues() + const [shouldPauseQueues, avgLatency] = await checkShouldPauseQueues() if (shouldPauseQueues) { let rssQueueCount: number | string = 'unknown' @@ -159,7 +157,7 @@ async function checkMetricsAndPauseQueues() { console.log('error fetching queue counts', err) } - const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` + const message = `Both queues have been paused due to API latency threshold exceedance (${avgLatency}).\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` await pauseQueues() await postToDiscord(message) From d8a5ed42e8d52c422fb8e67fdd41a208e36685d4 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:28:56 +0800 Subject: [PATCH 29/78] Remove unused var --- packages/queue-manager/src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 47383ba58..27d13e22b 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -79,8 +79,6 @@ const checkShouldPauseQueues = async () => { }, }) - let shouldPauseQueues = false - for (const ts of timeSeries) { // We only want to look at the backend service right now if ( From fd57e075d7ca112dbdb883b30845df11d4f7a083 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:30:57 +0800 Subject: [PATCH 30/78] Clean up return signature --- packages/queue-manager/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 27d13e22b..37124ce7b 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -97,12 +97,12 @@ const checkShouldPauseQueues = async () => { 0 ) / ts.points.length if (avgLatency > LATENCY_THRESHOLD) { - return [true, avgLatency] + return { shouldPauseQueues: true, avgLatency: avgLatency } } } } - return [false, 0] + return { shouldPauseQueues: false, avgLatency: 0 } } const getQueueTaskCount = async (queueName: string) => { @@ -143,7 +143,7 @@ async function checkMetricsAndPauseQueues() { throw new Error('environment not supplied.') } - const [shouldPauseQueues, avgLatency] = await checkShouldPauseQueues() + const { shouldPauseQueues, avgLatency } = await checkShouldPauseQueues() if (shouldPauseQueues) { let rssQueueCount: number | string = 'unknown' From 7f61acc907d6f9f3466650543d586ab649a1bfa2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 19:40:29 +0800 Subject: [PATCH 31/78] skip old feed if the pubDate is before last_fetched timestamp --- packages/rss-handler/src/index.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 9c37e22f9..60e789e73 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -122,6 +122,7 @@ const parser = new Parser({ maxRedirects: 10, customFields: { item: [['link', 'links', { keepArray: true }], 'published', 'updated'], + feed: ['dc:date', 'lastBuildDate', 'pubDate'], }, headers: { // some rss feeds require user agent @@ -198,7 +199,16 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( // fetch feed let itemCount = 0 const feed = await parser.parseURL(feedUrl) - console.log('Fetched feed', feed.title, new Date()) + console.log('Fetched feed', feed, new Date()) + + const feedPubDate = (feed['dc:date'] || + feed.pubDate || + feed.lastBuildDate) as string | undefined + console.log('Feed pub date', feedPubDate) + if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { + console.log('Skipping old feed', feedPubDate) + return res.send('ok') + } // save each item in the feed for (const item of feed.items) { From 2c830176685473e16539340962eba1084e349e6f Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 18 Oct 2023 03:00:40 +0900 Subject: [PATCH 32/78] Fix typo in ContentLoading.swift propogate -> propagate --- .../Sources/Services/DataService/ContentLoading.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift index f32e5005c..d276c40e1 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift @@ -231,7 +231,7 @@ extension DataService { _ = try await saveURL(id: id, url: url) } } catch { - // We don't propogate these errors, we just let it pass through so + // We don't propagate these errors, we just let it pass through so // the user can attempt to fetch content again. print("Error syncUnsyncedArticleContent", error) } From d315418171f68d849434221183d4c3893a522762 Mon Sep 17 00:00:00 2001 From: Paul Mullins Date: Tue, 17 Oct 2023 18:49:37 -0400 Subject: [PATCH 33/78] Fix password in readme, password is demo_password not demo, see packages/db/setup.sh for reference --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 54bf3b2b4..698528626 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ Open and confirm Omnivore is running ### 3. Login with the test account -During database setup docker compose creates an account `demo@omnivore.app`, password: `demo`. +During database setup docker compose creates an account `demo@omnivore.app`, password: `demo_password`. Go to in your browser and choose `Continue with Email` to login. From 1053c9b53a719627c62b7dccb1d811641427b8c3 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 09:11:11 +0800 Subject: [PATCH 34/78] Bump threshold for testing --- packages/queue-manager/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 37124ce7b..9574d0684 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -30,7 +30,7 @@ if ( const LATENCY_THRESHOLD = 500 const RSS_QUEUE_THRESHOLD = 20_000 -const IMPORT_QUEUE_THRESHOLD = 200_000 +const IMPORT_QUEUE_THRESHOLD = 250_000 const postToDiscord = async (message: string) => { console.log('notify message', { message }) From 25c0051fd4ab807d2ad770d7164ed9f951252497 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:14:35 +0800 Subject: [PATCH 35/78] Only use signed URLs for PDF attachments --- packages/api/src/resolvers/article/index.ts | 6 +----- packages/api/src/routers/svc/email_attachment.ts | 6 +----- packages/api/src/utils/uploads.ts | 16 ---------------- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index bf7dc89f9..bd016a345 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -98,10 +98,7 @@ import { parsePreparedContent, } from '../../utils/parser' import { parseSearchQuery, sortParamsToSort } from '../../utils/search' -import { - getStorageFileDetails, - makeStorageFilePublic, -} from '../../utils/uploads' +import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' export enum ArticleFormat { @@ -310,7 +307,6 @@ export const createArticleResolver = authorized< pubsub ) } - await makeStorageFilePublic(uploadFileData.id, uploadFileData.fileName) } let libraryItemToReturn: LibraryItem diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index b9bf3c77c..926317f4e 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -143,11 +143,7 @@ export function emailAttachmentRouter() { return res.status(400).send('BAD REQUEST') } - const uploadFileUrlOverride = await makeStorageFilePublic( - uploadFileData.id, - uploadFileData.fileName - ) - + const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFileId}/${uploadFile.fileName}` const uploadFileHash = uploadFileDetails.md5Hash const itemType = uploadFile.contentType === 'application/pdf' diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 34bfb5854..1cfeb1ff8 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -81,22 +81,6 @@ export const generateDownloadSignedUrl = async ( return url } -export const makeStorageFilePublic = async ( - id: string, - fileName: string -): Promise => { - // if (env.dev.isLocal) { - // return 'http://localhost:3000/public/' + id + '/' + fileName - // } - - // Makes the file public - const filePathName = generateUploadFilePathName(id, fileName) - await storage.bucket(bucketName).file(filePathName).makePublic() - - const fileObj = storage.bucket(bucketName).file(filePathName) - return fileObj.publicUrl() -} - export const getStorageFileDetails = async ( id: string, fileName: string From b6ce4f39b1de907de48d08b7ad514dde6c8e0780 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:15:49 +0800 Subject: [PATCH 36/78] Fix imports --- packages/api/src/routers/svc/email_attachment.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index 926317f4e..096359673 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -24,7 +24,6 @@ import { generateUploadFilePathName, generateUploadSignedUrl, getStorageFileDetails, - makeStorageFilePublic, } from '../../utils/uploads' export function emailAttachmentRouter() { From 4220922ff80db3b57ea41f7ccbc5ed224ec86c44 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:23:26 +0800 Subject: [PATCH 37/78] Preserve original URL if possible --- packages/api/src/resolvers/upload_files/index.ts | 6 +----- packages/api/src/utils/uploads.ts | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 36169b5c4..643d758d4 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -110,13 +110,9 @@ export const uploadFileRequestResolver = authorized< input.contentType ) - const publicUrl = getFilePublicUrl(uploadFilePathName) - - // If this is a file URL, we swap in the GCS public URL if (isFileUrl(input.url)) { await authTrx(async (tx) => { await tx.getRepository(UploadFile).update(uploadFileId, { - url: publicUrl, status: UploadFileStatus.Initialized, }) }) @@ -142,7 +138,7 @@ export const uploadFileRequestResolver = authorized< const uploadFileId = uploadFileData.id const item = await createLibraryItem( { - originalUrl: isFileUrl(input.url) ? publicUrl : input.url, + originalUrl: input.url, id: input.clientRequestId || undefined, user: { id: uid }, title, diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 1cfeb1ff8..2d31b525f 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -34,10 +34,6 @@ const storage = env.fileUpload?.gcsUploadSAKeyFilePath : new Storage() const bucketName = env.fileUpload.gcsUploadBucket -export const getFilePublicUrl = (filePathName: string): string => { - return storage.bucket(bucketName).file(filePathName).publicUrl() -} - export const countOfFilesWithPrefix = async (prefix: string) => { const [files] = await storage.bucket(bucketName).getFiles({ prefix }) return files.length From 78843fff14558a12912b85fe402269eabcb9e746 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:27:09 +0800 Subject: [PATCH 38/78] Remove import --- packages/api/src/resolvers/upload_files/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 643d758d4..58eca5370 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -24,7 +24,6 @@ import { contentReaderForLibraryItem, generateUploadFilePathName, generateUploadSignedUrl, - getFilePublicUrl, } from '../../utils/uploads' const isFileUrl = (url: string): boolean => { From b88c64424f48954e37f3477db6e2ae5fd3fd41a3 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:37:21 +0800 Subject: [PATCH 39/78] Special handling of file URLs --- packages/api/src/resolvers/upload_files/index.ts | 3 +++ packages/api/src/routers/svc/email_attachment.ts | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 58eca5370..93e622ea0 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -109,9 +109,12 @@ export const uploadFileRequestResolver = authorized< input.contentType ) + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` if (isFileUrl(input.url)) { await authTrx(async (tx) => { await tx.getRepository(UploadFile).update(uploadFileId, { + url: attachmentUrl, status: UploadFileStatus.Initialized, }) }) diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index 096359673..6c127135b 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -142,7 +142,12 @@ export function emailAttachmentRouter() { return res.status(400).send('BAD REQUEST') } - const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFileId}/${uploadFile.fileName}` + const uploadFilePathName = generateUploadFilePathName( + uploadFileId, + uploadFile.fileName + ) + + const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFilePathName}` const uploadFileHash = uploadFileDetails.md5Hash const itemType = uploadFile.contentType === 'application/pdf' From 71aaef3628b8650da3ef56641b4c8cb79d5b286d Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:43:53 +0800 Subject: [PATCH 40/78] Update attachment URL --- packages/api/src/resolvers/upload_files/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 93e622ea0..9ea214584 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -140,8 +140,8 @@ export const uploadFileRequestResolver = authorized< const uploadFileId = uploadFileData.id const item = await createLibraryItem( { - originalUrl: input.url, id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, user: { id: uid }, title, readableContent: '', From 0874f6ff21fb0489ad222ca65e9412dc3b360aca Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 22:12:38 +0800 Subject: [PATCH 41/78] log x-forwarded-for header for debugging purpose --- packages/api/src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index cc757acc3..c36d25148 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -72,6 +72,7 @@ export const createApp = (): { } }, keyGenerator: (req) => { + console.log('x-forwarded-for header', req.header('x-forwarded-for')) return getTokenByRequest(req) || req.ip }, // skip preflight requests and test requests From 91076014cded16d70449a50dba807a748ca881c2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 22:47:41 +0800 Subject: [PATCH 42/78] log req.ip --- packages/api/src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index c36d25148..14004e642 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -73,6 +73,7 @@ export const createApp = (): { }, keyGenerator: (req) => { console.log('x-forwarded-for header', req.header('x-forwarded-for')) + console.log('ip', req.ip) return getTokenByRequest(req) || req.ip }, // skip preflight requests and test requests From 32ba744d20802e24a2ab0fdedb1c05267026ab20 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 18 Oct 2023 10:59:31 +0800 Subject: [PATCH 43/78] test trust proxy=1 --- packages/api/src/server.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 14004e642..0c9456934 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -57,6 +57,7 @@ export const createApp = (): { app.use(cookieParser()) app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) + app.set('trust proxy', 1) const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute @@ -72,8 +73,8 @@ export const createApp = (): { } }, keyGenerator: (req) => { - console.log('x-forwarded-for header', req.header('x-forwarded-for')) - console.log('ip', req.ip) + console.log('x-forwarded-for header:', req.header('x-forwarded-for')) + console.log('ip:', req.ip) return getTokenByRequest(req) || req.ip }, // skip preflight requests and test requests From 506004d410de99fcc1bd0b1100ff22416d533be2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 18 Oct 2023 11:58:15 +0800 Subject: [PATCH 44/78] test trust proxy=2 --- packages/api/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 0c9456934..dbad3ec94 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -57,7 +57,7 @@ export const createApp = (): { app.use(cookieParser()) app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) - app.set('trust proxy', 1) + app.set('trust proxy', 2) const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute From 988b20e30caf8d60390aaff239957c1b04fd9dd3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 18 Oct 2023 12:30:27 +0800 Subject: [PATCH 45/78] test trust proxy=true --- packages/api/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index dbad3ec94..00c7d0e09 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -57,7 +57,7 @@ export const createApp = (): { app.use(cookieParser()) app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) - app.set('trust proxy', 2) + app.set('trust proxy', true) const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute From 46313b14ba1fd4d5dcf03c9a08acb104b17fcbc5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 18 Oct 2023 13:06:33 +0800 Subject: [PATCH 46/78] set trust proxy = true if set in env var --- packages/api/src/server.ts | 6 +++--- packages/api/src/util.ts | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 00c7d0e09..9e41af64d 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -57,7 +57,9 @@ export const createApp = (): { app.use(cookieParser()) app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) - app.set('trust proxy', true) + + // set to true if behind a reverse proxy/load balancer + app.set('trust proxy', env.server.trustProxy) const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute @@ -73,8 +75,6 @@ export const createApp = (): { } }, keyGenerator: (req) => { - console.log('x-forwarded-for header:', req.header('x-forwarded-for')) - console.log('ip:', req.ip) return getTokenByRequest(req) || req.ip }, // skip preflight requests and test requests diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 71df1cbc5..5764307a2 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -21,6 +21,7 @@ interface BackendEnv { gateway_url: string apiEnv: string instanceId: string + trustProxy: boolean } client: { url: string @@ -159,6 +160,7 @@ const nullableEnvVars = [ 'RSS_FEED_TASK_HANDLER_URL', 'SENDGRID_VERIFICATION_TEMPLATE_ID', 'REMINDER_TASK_HANDLER_URL', + 'TRUST_PROXY', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -207,6 +209,7 @@ export function getEnv(): BackendEnv { apiEnv: parse('API_ENV'), instanceId: parse('GAE_INSTANCE') || `x${os.userInfo().username}_${os.hostname()}`, + trustProxy: parse('TRUST_PROXY') === 'true', } const client = { url: parse('CLIENT_URL'), From cf101c6d18e39f252d7b2b0355803201ca0b96be Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:26:03 +0800 Subject: [PATCH 47/78] Cache and check feed checksums to reduce fetching --- packages/api/src/entity/subscription.ts | 3 + packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + .../api/src/resolvers/subscriptions/index.ts | 1 + packages/api/src/routers/svc/rss_feed.ts | 2 +- packages/api/src/schema.ts | 1 + packages/api/src/utils/createTask.ts | 1 + ...do.add_checksum_to_subscriptions_table.sql | 7 ++ ...do.add_checksum_to_subscriptions_table.sql | 9 ++ packages/rss-handler/src/index.ts | 84 +++++++++++++------ 10 files changed, 85 insertions(+), 25 deletions(-) create mode 100755 packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql create mode 100755 packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index 90f7d593b..b63400d56 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -59,6 +59,9 @@ export class Subscription { @Column('timestamp', { nullable: true }) lastFetchedAt?: Date | null + @Column('text', { nullable: true }) + lastFetchedChecksum?: string | null + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) createdAt!: Date diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index f0f41ff68..8ceb89481 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2978,6 +2978,7 @@ export type UpdateSubscriptionInput = { description?: InputMaybe; id: Scalars['ID']; lastFetchedAt?: InputMaybe; + lastfetchedChecksum?: InputMaybe; name?: InputMaybe; status?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 3ef33f886..39a6c00e7 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2391,6 +2391,7 @@ input UpdateSubscriptionInput { description: String id: ID! lastFetchedAt: Date + lastfetchedChecksum: String name: String status: SubscriptionStatus } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 74375441d..a1069423b 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -290,6 +290,7 @@ export const updateSubscriptionResolver = authorized< lastFetchedAt: input.lastFetchedAt ? new Date(input.lastFetchedAt) : undefined, + lastFetchedChecksum: input.lastfetchedChecksum, status: input.status || undefined, }) diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 0834e5025..c06ec9f71 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -24,7 +24,7 @@ export function rssFeedRouter() { // get all active rss feed subscriptions const subscriptions = await getRepository(Subscription).find({ - select: ['id', 'url', 'user', 'lastFetchedAt'], + select: ['id', 'url', 'user', 'lastFetchedAt', 'lastFetchedChecksum'], where: { type: SubscriptionType.Rss, status: SubscriptionStatus.Active, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e3b2b3442..6d78471cb 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2548,6 +2548,7 @@ const schema = gql` name: String description: String lastFetchedAt: Date + lastfetchedChecksum: String status: SubscriptionStatus } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index bb3457e00..483921bc9 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -601,6 +601,7 @@ export const enqueueRssFeedFetch = async ( subscriptionId: rssFeedSubscription.id, feedUrl: rssFeedSubscription.url, lastFetchedAt: rssFeedSubscription.lastFetchedAt?.getTime() || 0, // unix timestamp in milliseconds + lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, } const headers = { diff --git a/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql new file mode 100755 index 000000000..5f704a666 --- /dev/null +++ b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql @@ -0,0 +1,7 @@ +-- Type: DO +-- Name: add_checksum_to_subscriptions_table +-- Description: Add a last fetched checksum field to the subscriptions table + +BEGIN; + +COMMIT; diff --git a/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql b/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql new file mode 100755 index 000000000..e138d501a --- /dev/null +++ b/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_checksum_to_subscriptions_table +-- Description: Add a last fetched checksum field to the subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP COLUMN last_fetched_checksum ; + +COMMIT; diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 60e789e73..4edd58709 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -1,5 +1,6 @@ import * as Sentry from '@sentry/serverless' import axios from 'axios' +import crypto from 'crypto' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import * as jwt from 'jsonwebtoken' import Parser, { Item } from 'rss-parser' @@ -10,6 +11,7 @@ interface RssFeedRequest { subscriptionId: string feedUrl: string lastFetchedAt: number // unix timestamp in milliseconds + lastFetchedChecksum: string | undefined } // link can be a string or an object @@ -21,10 +23,42 @@ function isRssFeedRequest(body: any): body is RssFeedRequest { ) } +type FeedFetchResult = { + url: string + content: string + checksum: string +} + +async function fetchAndChecksum(url: string): Promise { + try { + // Fetch the content from the URL + const response = await axios.get(url, { + responseType: 'arraybuffer', + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + Accept: + 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', + }, + }) + + // Create a sha256 hash of the content + const hash = crypto.createHash('sha256') + hash.update(response.data) + + return { url, content: response.data, checksum: hash.digest('hex') } + } catch (error) { + throw new Error( + `Failed to fetch or hash content from ${url}. Error: ${error}` + ) + } +} + const sendUpdateSubscriptionMutation = async ( userId: string, subscriptionId: string, - lastFetchedAt: Date + lastFetchedAt: Date, + lastFetchedChecksum: string ) => { const JWT_SECRET = process.env.JWT_SECRET const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT @@ -51,6 +85,7 @@ const sendUpdateSubscriptionMutation = async ( input: { id: subscriptionId, lastFetchedAt, + lastFetchedChecksum, }, }, }) @@ -121,15 +156,12 @@ const parser = new Parser({ timeout: 60000, // 60 seconds maxRedirects: 10, customFields: { - item: [['link', 'links', { keepArray: true }], 'published', 'updated'], - feed: ['dc:date', 'lastBuildDate', 'pubDate'], - }, - headers: { - // some rss feeds require user agent - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - Accept: - 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', + item: [ + ['link', 'links', { keepArray: true }], + 'published', + 'updated', + 'created', + ], }, }) @@ -190,31 +222,34 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( return res.status(400).send('INVALID_REQUEST_BODY') } - const { feedUrl, subscriptionId, lastFetchedAt } = req.body + const { feedUrl, subscriptionId, lastFetchedAt, lastFetchedChecksum } = + req.body console.log('Processing feed', feedUrl, lastFetchedAt) let lastItemFetchedAt: Date | null = null let lastValidItem: Item | null = null + let updatedLastFetchedChecksum: string | null + + let fetchResult = await fetchAndChecksum(feedUrl) + if (fetchResult.checksum === lastFetchedChecksum) { + console.log('feed has not been updated', feedUrl, lastFetchedChecksum) + return res.status(200) + } + updatedLastFetchedChecksum = fetchResult.checksum // fetch feed let itemCount = 0 - const feed = await parser.parseURL(feedUrl) - console.log('Fetched feed', feed, new Date()) - - const feedPubDate = (feed['dc:date'] || - feed.pubDate || - feed.lastBuildDate) as string | undefined - console.log('Feed pub date', feedPubDate) - if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { - console.log('Skipping old feed', feedPubDate) - return res.send('ok') - } + const feed = await parser.parseString(fetchResult.content) + console.log('Fetched feed', feed.title, new Date()) // save each item in the feed for (const item of feed.items) { // use published or updated if isoDate is not available for atom feeds item.isoDate = - item.isoDate || (item.published as string) || (item.updated as string) + item.isoDate || + (item.published as string) || + (item.updated as string) || + (item.created as string) console.log('Processing feed item', item.links, item.isoDate) if (!item.links || item.links.length === 0) { @@ -299,7 +334,8 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( const updatedSubscription = await sendUpdateSubscriptionMutation( userId, subscriptionId, - lastItemFetchedAt + lastItemFetchedAt, + updatedLastFetchedChecksum ) console.log('Updated subscription', updatedSubscription) From f14fc034fe95ed20631c98b8e2c56691d759d8b7 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:28:22 +0800 Subject: [PATCH 48/78] Add last fetched column to the database --- .../migrations/0138.do.add_checksum_to_subscriptions_table.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql index 5f704a666..dfc37edfe 100755 --- a/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql +++ b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql @@ -4,4 +4,6 @@ BEGIN; +ALTER TABLE omnivore.subscriptions ADD COLUMN last_fetched_checksum TEXT ; + COMMIT; From 78484136454b64f3f3ace2b21e48e26532920e78 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:34:20 +0800 Subject: [PATCH 49/78] Update schema --- packages/api/src/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 6d78471cb..2cc4e8b58 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2548,7 +2548,7 @@ const schema = gql` name: String description: String lastFetchedAt: Date - lastfetchedChecksum: String + lastFetchedChecksum: String status: SubscriptionStatus } From d592dc44c6a6b931e5757dfddde1ae998aa55b83 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:34:40 +0800 Subject: [PATCH 50/78] Add back missing pubdate check --- packages/rss-handler/src/index.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 4edd58709..048592c27 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -242,6 +242,15 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( const feed = await parser.parseString(fetchResult.content) console.log('Fetched feed', feed.title, new Date()) + const feedPubDate = (feed['dc:date'] || + feed.pubDate || + feed.lastBuildDate) as string | undefined + console.log('Feed pub date', feedPubDate) + if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { + console.log('Skipping old feed', feedPubDate) + return res.send('ok') + } + // save each item in the feed for (const item of feed.items) { // use published or updated if isoDate is not available for atom feeds From 921a46a13ae7828823f7513093dfae1d025681f2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:51:23 +0800 Subject: [PATCH 51/78] Linting fixes --- packages/rss-handler/src/index.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 048592c27..5d1a3c654 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -31,7 +31,6 @@ type FeedFetchResult = { async function fetchAndChecksum(url: string): Promise { try { - // Fetch the content from the URL const response = await axios.get(url, { responseType: 'arraybuffer', headers: { @@ -42,15 +41,15 @@ async function fetchAndChecksum(url: string): Promise { }, }) - // Create a sha256 hash of the content const hash = crypto.createHash('sha256') - hash.update(response.data) + hash.update(response.data as Buffer) - return { url, content: response.data, checksum: hash.digest('hex') } + const dataStr = (response.data as Buffer).toString() + + return { url, content: dataStr, checksum: hash.digest('hex') } } catch (error) { - throw new Error( - `Failed to fetch or hash content from ${url}. Error: ${error}` - ) + console.log(error) + throw new Error(`Failed to fetch or hash content from ${url}.`) } } @@ -228,14 +227,13 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( let lastItemFetchedAt: Date | null = null let lastValidItem: Item | null = null - let updatedLastFetchedChecksum: string | null - let fetchResult = await fetchAndChecksum(feedUrl) + const fetchResult = await fetchAndChecksum(feedUrl) if (fetchResult.checksum === lastFetchedChecksum) { console.log('feed has not been updated', feedUrl, lastFetchedChecksum) return res.status(200) } - updatedLastFetchedChecksum = fetchResult.checksum + const updatedLastFetchedChecksum = fetchResult.checksum // fetch feed let itemCount = 0 From 479fc8fb7d96bdec37505dcea683593e6f051de2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 17:11:02 +0800 Subject: [PATCH 52/78] Add a test for checksumming --- packages/rss-handler/package.json | 3 ++- packages/rss-handler/src/index.ts | 8 +------- packages/rss-handler/test/checksum.test.ts | 14 ++++++++++++++ packages/rss-handler/test/stub.test.ts | 8 -------- 4 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 packages/rss-handler/test/checksum.test.ts delete mode 100644 packages/rss-handler/test/stub.test.ts diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json index 2a6d40851..0db5d93fd 100644 --- a/packages/rss-handler/package.json +++ b/packages/rss-handler/package.json @@ -17,7 +17,8 @@ "devDependencies": { "chai": "^4.3.6", "eslint-plugin-prettier": "^4.0.0", - "mocha": "^10.0.0" + "mocha": "^10.0.0", + "nock": "^13.3.4" }, "dependencies": { "@google-cloud/functions-framework": "3.1.2", diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 5d1a3c654..2b1bdf1fc 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -23,13 +23,7 @@ function isRssFeedRequest(body: any): body is RssFeedRequest { ) } -type FeedFetchResult = { - url: string - content: string - checksum: string -} - -async function fetchAndChecksum(url: string): Promise { +export const fetchAndChecksum = async (url: string) => { try { const response = await axios.get(url, { responseType: 'arraybuffer', diff --git a/packages/rss-handler/test/checksum.test.ts b/packages/rss-handler/test/checksum.test.ts new file mode 100644 index 000000000..24eee7a0b --- /dev/null +++ b/packages/rss-handler/test/checksum.test.ts @@ -0,0 +1,14 @@ +import 'mocha' +import nock from 'nock' +import { expect } from 'chai' +import { fetchAndChecksum } from '../src/index' + +describe('fetchAndChecksum', () => { + it('should hash the content available', async () => { + nock('https://fake.com', {}).get('/rss.xml').reply(200, 'i am some content') + const result = await fetchAndChecksum('https://fake.com/rss.xml') + expect(result.checksum).to.eq( + 'd6bc10faec048d999d0cf4b2f7103d84557fb9cd94c3bccd17884b1288949375' + ) + }) +}) diff --git a/packages/rss-handler/test/stub.test.ts b/packages/rss-handler/test/stub.test.ts deleted file mode 100644 index 24ad25c8f..000000000 --- a/packages/rss-handler/test/stub.test.ts +++ /dev/null @@ -1,8 +0,0 @@ -import 'mocha' -import { expect } from 'chai' - -describe('stub test', () => { - it('should pass', () => { - expect(true).to.be.true - }) -}) From d2c0efc540142d46c4df639b946bddfd03331d61 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 18:22:22 +0800 Subject: [PATCH 53/78] add some debug --- packages/rss-handler/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2b1bdf1fc..a61bdfbec 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -83,6 +83,8 @@ const sendUpdateSubscriptionMutation = async ( }, }) + console.log('sending', data) + const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string try { const response = await axios.post( From fbaaaeca7c67714d9ce7189994ad8fd6d7ae8e7b Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 18:48:18 +0800 Subject: [PATCH 54/78] Add some debugging --- packages/api/src/resolvers/subscriptions/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index a1069423b..8cc6c4d01 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -290,7 +290,7 @@ export const updateSubscriptionResolver = authorized< lastFetchedAt: input.lastFetchedAt ? new Date(input.lastFetchedAt) : undefined, - lastFetchedChecksum: input.lastfetchedChecksum, + lastFetchedChecksum: input.lastfetchedChecksum || undefined, status: input.status || undefined, }) @@ -300,6 +300,8 @@ export const updateSubscriptionResolver = authorized< }) }) + console.log('updatedSubscription', updatedSubscription) + return { subscription: updatedSubscription, } From 5c576347a8878701c7debe114419ca6173e2f724 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 19:26:45 +0800 Subject: [PATCH 55/78] MOre debug --- packages/api/src/resolvers/subscriptions/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 8cc6c4d01..6b415e4d4 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -282,8 +282,7 @@ export const updateSubscriptionResolver = authorized< const updatedSubscription = await authTrx(async (t) => { const repo = t.getRepository(Subscription) - // update subscription - await t.getRepository(Subscription).save({ + const dict = { id: input.id, name: input.name || undefined, description: input.description || undefined, @@ -292,7 +291,10 @@ export const updateSubscriptionResolver = authorized< : undefined, lastFetchedChecksum: input.lastfetchedChecksum || undefined, status: input.status || undefined, - }) + } + console.log('saving dict:', JSON.stringify(dict)) + // update subscription + await t.getRepository(Subscription).save(dict) return repo.findOneByOrFail({ id: input.id, From 0fcc7096aa9598bacb88eb2ce22fae4a4e9f905d Mon Sep 17 00:00:00 2001 From: Surav Shrestha Date: Wed, 18 Oct 2023 17:33:22 +0545 Subject: [PATCH 56/78] docs: fix typo in packages/puppeteer-parse/README.md --- packages/puppeteer-parse/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/README.md b/packages/puppeteer-parse/README.md index 6155c45b0..373c57353 100644 --- a/packages/puppeteer-parse/README.md +++ b/packages/puppeteer-parse/README.md @@ -1,3 +1,3 @@ # Puppeteer parsing function handler -This workspace is used to provide the module for the app to hande requests for the article parsing via Puppeteer. +This workspace is used to provide the module for the app to handle requests for the article parsing via Puppeteer. From 78a642db6d0142628e387376b2ed08ce17d32ee1 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 19:53:26 +0800 Subject: [PATCH 57/78] Update type signature --- packages/api/src/generated/graphql.ts | 2 +- packages/api/src/generated/schema.graphql | 2 +- packages/api/src/resolvers/subscriptions/index.ts | 10 ++++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 8ceb89481..0ad412b4f 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2978,7 +2978,7 @@ export type UpdateSubscriptionInput = { description?: InputMaybe; id: Scalars['ID']; lastFetchedAt?: InputMaybe; - lastfetchedChecksum?: InputMaybe; + lastFetchedChecksum?: InputMaybe; name?: InputMaybe; status?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 39a6c00e7..af9064bee 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2391,7 +2391,7 @@ input UpdateSubscriptionInput { description: String id: ID! lastFetchedAt: Date - lastfetchedChecksum: String + lastFetchedChecksum: String name: String status: SubscriptionStatus } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 6b415e4d4..50fdcb839 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -282,19 +282,17 @@ export const updateSubscriptionResolver = authorized< const updatedSubscription = await authTrx(async (t) => { const repo = t.getRepository(Subscription) - const dict = { + // update subscription + await t.getRepository(Subscription).save({ id: input.id, name: input.name || undefined, description: input.description || undefined, lastFetchedAt: input.lastFetchedAt ? new Date(input.lastFetchedAt) : undefined, - lastFetchedChecksum: input.lastfetchedChecksum || undefined, + lastFetchedChecksum: input.lastFetchedChecksum || undefined, status: input.status || undefined, - } - console.log('saving dict:', JSON.stringify(dict)) - // update subscription - await t.getRepository(Subscription).save(dict) + }) return repo.findOneByOrFail({ id: input.id, From 3a4547a6afbf391e6ae00dca92271623819a9d4a Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 21:01:01 +0800 Subject: [PATCH 58/78] Remove debug --- packages/rss-handler/src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index a61bdfbec..2b1bdf1fc 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -83,8 +83,6 @@ const sendUpdateSubscriptionMutation = async ( }, }) - console.log('sending', data) - const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string try { const response = await axios.post( From c93e61ec247745ac44fb0356e2db878de3afcada Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 21:30:23 +0800 Subject: [PATCH 59/78] Remove debug --- packages/api/src/resolvers/subscriptions/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 50fdcb839..430347654 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -300,8 +300,6 @@ export const updateSubscriptionResolver = authorized< }) }) - console.log('updatedSubscription', updatedSubscription) - return { subscription: updatedSubscription, } From 4b171c0657ffc85e1a43d69279b341b582173a10 Mon Sep 17 00:00:00 2001 From: Surav Shrestha Date: Wed, 18 Oct 2023 20:35:35 +0545 Subject: [PATCH 60/78] docs: fix typos in packages/content-fetch/README.md --- packages/content-fetch/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/content-fetch/README.md b/packages/content-fetch/README.md index 501c004ee..c72e5720f 100644 --- a/packages/content-fetch/README.md +++ b/packages/content-fetch/README.md @@ -1,6 +1,6 @@ # Puppeteer parsing function handler -This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. +This workspace is used to provide the GCF for the app to handle requests for the article parsing via Puppeteer. ## Using locally @@ -8,7 +8,7 @@ Copy .env.example file to .env file: `cp .env.example .env` Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). -After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) +After this, you should be able to access the function on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) ## Deployment From 699e8077dcfd738f0fa5c252b1165c68eb259053 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 19 Oct 2023 09:57:28 +0800 Subject: [PATCH 61/78] Add feed custom fields, set timeout/redirects --- packages/rss-handler/src/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2b1bdf1fc..ca6afb04b 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -27,6 +27,8 @@ export const fetchAndChecksum = async (url: string) => { try { const response = await axios.get(url, { responseType: 'arraybuffer', + timeout: 60_000, + maxRedirects: 10, headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', @@ -146,8 +148,6 @@ Sentry.GCPFunction.init({ const signToken = promisify(jwt.sign) const parser = new Parser({ - timeout: 60000, // 60 seconds - maxRedirects: 10, customFields: { item: [ ['link', 'links', { keepArray: true }], @@ -155,6 +155,7 @@ const parser = new Parser({ 'updated', 'created', ], + feed: ['dc:date', 'lastBuildDate', 'pubDate'], }, }) From 1733cf756dc96b6e6f6a7a48e7b556dce28f0dd5 Mon Sep 17 00:00:00 2001 From: h3n4l Date: Thu, 19 Oct 2023 11:42:42 +0800 Subject: [PATCH 62/78] docs: add native ios app source code link in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 698528626..111cdb065 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ We built Omnivore because we love reading and we want it to be more social. Join - Add newsletter articles via email (with substack support!) - PDF support - [Web app](https://omnivore.app/) written in Node.js and TypeScript -- [Native iOS app](https://omnivore.app/install/ios) +- [Native iOS app](https://omnivore.app/install/ios) ([source](https://github.com/omnivore-app/omnivore/tree/main/apple)) - [Android app](https://omnivore.app/install/android) ([source](https://github.com/omnivore-app/omnivore/tree/main/android/Omnivore)) - Progressive web app for Android users - Browser extensions for [Chrome](https://omnivore.app/install/chrome), [Safari](https://omnivore.app/install/safari), [Firefox](https://omnivore.app/install/firefox), and [Edge](https://omnivore.app/install/edge) From 88a46e6ee67c2582aa341f47fc5f1e76b2baf212 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 19 Oct 2023 14:18:01 +0800 Subject: [PATCH 63/78] Send ok when checksum has not been updated --- packages/rss-handler/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index ca6afb04b..8b3021659 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -226,7 +226,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( const fetchResult = await fetchAndChecksum(feedUrl) if (fetchResult.checksum === lastFetchedChecksum) { console.log('feed has not been updated', feedUrl, lastFetchedChecksum) - return res.status(200) + return res.send('ok') } const updatedLastFetchedChecksum = fetchResult.checksum From 18aece84e78d1769fadfc67f632d3030fa25535b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 10:24:42 +0800 Subject: [PATCH 64/78] add scheduledAt to subscriptions --- packages/api/src/entity/subscription.ts | 3 +++ .../0138.do.add_scheduled_at_to_subscription.sql | 9 +++++++++ .../0138.undo.add_scheduled_at_to_subscription.sql | 9 +++++++++ 3 files changed, 21 insertions(+) create mode 100755 packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql create mode 100755 packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index b63400d56..fea63d890 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -67,4 +67,7 @@ export class Subscription { @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) updatedAt!: Date + + @Column('timestamp', { nullable: true }) + scheduledAt?: Date | null } diff --git a/packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql new file mode 100755 index 000000000..c45201a06 --- /dev/null +++ b/packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: add_scheduled_at_to_subscription +-- Description: Add scheduled_at field to omnivore.subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions ADD COLUMN scheduled_at timestamptz; + +COMMIT; diff --git a/packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql new file mode 100755 index 000000000..090f63ac4 --- /dev/null +++ b/packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_scheduled_at_to_subscription +-- Description: Add scheduled_at field to omnivore.subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP COLUMN IF EXISTS scheduled_at; + +COMMIT; From 5283a8490873aca5d636f6b1bbce77bb98c3f203 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 11:35:58 +0800 Subject: [PATCH 65/78] add scheduledAt to the task payload --- packages/api/src/services/subscriptions.ts | 16 ++- packages/api/src/utils/createTask.ts | 1 + .../api/test/resolvers/subscriptions.test.ts | 98 ++++++++++++++++++- 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 8aaea89b8..3a30469a8 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -1,4 +1,5 @@ import axios from 'axios' +import { DeleteResult } from 'typeorm' import { appDataSource } from '../data_source' import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' @@ -189,7 +190,8 @@ export const createSubscription = async ( newsletterEmail?: NewsletterEmail, status = SubscriptionStatus.Active, unsubscribeMailTo?: string, - subscriptionType = SubscriptionType.Newsletter + subscriptionType = SubscriptionType.Newsletter, + url?: string ): Promise => { return getRepository(Subscription).save({ user: { id: userId }, @@ -199,5 +201,17 @@ export const createSubscription = async ( unsubscribeMailTo, lastFetchedAt: new Date(), type: subscriptionType, + url, }) } + +export const deleteSubscription = async ( + id: string, + userId: string +): Promise => { + return authTrx( + (tx) => tx.getRepository(Subscription).delete(id), + undefined, + userId + ) +} diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 483921bc9..f7cc4c829 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -602,6 +602,7 @@ export const enqueueRssFeedFetch = async ( feedUrl: rssFeedSubscription.url, lastFetchedAt: rssFeedSubscription.lastFetchedAt?.getTime() || 0, // unix timestamp in milliseconds lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, + scheduledAt: new Date().getTime(), // unix timestamp in milliseconds } const headers = { diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 7a9db963d..22ee107b1 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -1,5 +1,6 @@ import chai, { expect } from 'chai' import 'mocha' +import Parser from 'rss-parser' import sinon from 'sinon' import sinonChai from 'sinon-chai' import { NewsletterEmail } from '../../src/entity/newsletter_email' @@ -9,13 +10,13 @@ import { SubscriptionStatus, SubscriptionType, } from '../../src/generated/graphql' -import { - createSubscription, - unsubscribe, - UNSUBSCRIBE_EMAIL_TEXT -} from '../../src/services/subscriptions' import { getRepository } from '../../src/repository' import { createNewsletterEmail } from '../../src/services/newsletters' +import { + createSubscription, + deleteSubscription, + UNSUBSCRIBE_EMAIL_TEXT, +} from '../../src/services/subscriptions' import { deleteUser } from '../../src/services/user' import * as sendEmail from '../../src/utils/sendEmail' import { createTestUser } from '../db' @@ -347,4 +348,91 @@ describe('Subscriptions API', () => { await getRepository(Subscription).remove(subscription) }) }) + + describe('Subscribe API', () => { + const query = ( + name: string | null, + url: string | null, + subscriptionType: SubscriptionType | null + ) => ` + mutation { + subscribe(input: { + name: ${name ? `"${name}"` : null} + url: ${url ? `"${url}"` : null} + subscriptionType: ${subscriptionType} + }) { + ... on SubscribeSuccess { + subscriptions { + id + } + } + ... on SubscribeError { + errorCodes + } + } + } + ` + + context('when subscribing to a rss feed', () => { + const url = 'https://www.omnivore.app/rss' + const subscriptionType = SubscriptionType.Rss + + before(async () => { + // fake rss parser + sinon.replace(Parser.prototype, 'parseURL', sinon.fake.resolves({ + title: 'RSS Feed', + description: 'RSS Feed Description', + })) + }) + + after(() => { + sinon.restore() + }) + + context('when the user is subscribed to the feed', () => { + let existingSubscription: Subscription + + before(async () => { + existingSubscription = await createSubscription( + user.id, + 'RSS Feed', + undefined, + SubscriptionStatus.Active, + url, + subscriptionType, + url + ) + }) + + after(async () => { + await deleteSubscription(existingSubscription.id, user.id) + }) + + it('returns an error', async () => { + const res = await graphqlRequest( + query(null, url, subscriptionType), + authToken + ).expect(200) + expect(res.body.data.subscribe.errorCodes).to.eql([ + 'ALREADY_SUBSCRIBED', + ]) + }) + }) + + it('creates a rss subscription', async () => { + const res = await graphqlRequest( + query(null, url, subscriptionType), + authToken + ).expect(200) + expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) + expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') + + // clean up + await deleteSubscription( + res.body.data.subscribe.subscriptions[0].id, + user.id + ) + }) + }) + }) }) From f50d1c9cb92fb8ffecf2a3031d5595c6982fa6ef Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 13:41:09 +0800 Subject: [PATCH 66/78] Group RSS subscriptions by feed URL and create cloud task for the groups --- .../api/src/resolvers/subscriptions/index.ts | 12 ++- packages/api/src/routers/svc/rss_feed.ts | 40 ++++--- packages/api/src/services/subscriptions.ts | 8 +- packages/api/src/utils/createTask.ts | 47 ++++---- packages/api/test/routers/rss_feed.test.ts | 101 ++++++++++++++++++ 5 files changed, 173 insertions(+), 35 deletions(-) create mode 100644 packages/api/test/routers/rss_feed.test.ts diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 430347654..7bf145452 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -235,11 +235,19 @@ export const subscribeResolver = authorized< } } + const newSubscription = newSubscriptions[0] + // create a cloud task to fetch rss feed item for the new subscription - await enqueueRssFeedFetch(uid, newSubscriptions[0]) + await enqueueRssFeedFetch({ + user_ids: [uid], + url: input.url, + subscription_ids: [newSubscription.id], + scheduled_timestamps: [null], + last_fetched_timestamps: [null], + }) return { - subscriptions: newSubscriptions, + subscriptions: [newSubscription], } } diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index c06ec9f71..93077f6e4 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -4,7 +4,10 @@ import { Subscription } from '../../entity/subscription' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' import { readPushSubscription } from '../../pubsub' import { getRepository } from '../../repository' -import { enqueueRssFeedFetch } from '../../utils/createTask' +import { + enqueueRssFeedFetch, + RssSubscriptionGroup, +} from '../../utils/createTask' import { logger } from '../../utils/logger' export function rssFeedRouter() { @@ -22,21 +25,34 @@ export function rssFeedRouter() { return res.status(200).send('Expired') } - // get all active rss feed subscriptions - const subscriptions = await getRepository(Subscription).find({ - select: ['id', 'url', 'user', 'lastFetchedAt', 'lastFetchedChecksum'], - where: { - type: SubscriptionType.Rss, - status: SubscriptionStatus.Active, - }, - relations: ['user'], - }) + // get active rss feed subscriptions scheduled for fetch and group by feed url + const subscriptionGroups = (await getRepository(Subscription).query( + ` + SELECT + url, + ARRAY_AGG(id) AS subscription_ids, + ARRAY_AGG(user_id) AS user_ids, + ARRAY_AGG(last_fetched_at) AS last_fetched_timestamps, + ARRAY_AGG(scheduled_at) AS scheduled_timestamps + FROM + omnivore.subscriptions + WHERE + type = $1 + AND status = $2 + AND (scheduled_at <= NOW() OR scheduled_at IS NULL) + GROUP BY + url + `, + [SubscriptionType.Rss, SubscriptionStatus.Active] + )) as RssSubscriptionGroup[] + + logger.info('scheduledSubscriptions', subscriptionGroups) // create a cloud taks to fetch rss feed item for each subscription await Promise.all( - subscriptions.map((subscription) => { + subscriptionGroups.map((subscriptionGroup) => { try { - return enqueueRssFeedFetch(subscription.user.id, subscription) + return enqueueRssFeedFetch(subscriptionGroup) } catch (error) { logger.info('error creating rss feed fetch task', error) } diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 3a30469a8..9e6e78b37 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -1,5 +1,5 @@ import axios from 'axios' -import { DeleteResult } from 'typeorm' +import { DeepPartial, DeleteResult } from 'typeorm' import { appDataSource } from '../data_source' import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' @@ -215,3 +215,9 @@ export const deleteSubscription = async ( userId ) } + +export const createRssSubscriptions = async ( + subscriptions: DeepPartial[] +) => { + return getRepository(Subscription).save(subscriptions) +} diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index f7cc4c829..7e09968ea 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -7,7 +7,6 @@ import axios from 'axios' import { nanoid } from 'nanoid' import { DeepPartial } from 'typeorm' import { Recommendation } from '../entity/recommendation' -import { Subscription } from '../entity/subscription' import { env } from '../env' import { ArticleSavingRequestStatus, @@ -592,21 +591,29 @@ export const enqueueThumbnailTask = async ( return createdTasks[0].name } -export const enqueueRssFeedFetch = async ( - userId: string, - rssFeedSubscription: Subscription -): Promise => { - const { GOOGLE_CLOUD_PROJECT } = process.env - const payload = { - subscriptionId: rssFeedSubscription.id, - feedUrl: rssFeedSubscription.url, - lastFetchedAt: rssFeedSubscription.lastFetchedAt?.getTime() || 0, // unix timestamp in milliseconds - lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, - scheduledAt: new Date().getTime(), // unix timestamp in milliseconds - } +export interface RssSubscriptionGroup { + url: string + subscription_ids: string[] + user_ids: string[] + last_fetched_timestamps: (Date | null)[] + scheduled_timestamps: (Date | null)[] +} - const headers = { - [OmnivoreAuthorizationHeader]: generateVerificationToken({ id: userId }), +export const enqueueRssFeedFetch = async ( + subscriptionGroup: RssSubscriptionGroup +): Promise => { + const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env + const payload = { + subscriptionIds: subscriptionGroup.subscription_ids, + feedUrl: subscriptionGroup.url, + lastFetchedTimestamps: subscriptionGroup.last_fetched_timestamps.map( + (timestamp) => timestamp?.getTime() || 0 + ), // unix timestamp in milliseconds + lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, + scheduledTimestamps: subscriptionGroup.scheduled_timestamps.map( + (timestamp) => timestamp?.getTime() || 0 + ), // unix timestamp in milliseconds + userIds: subscriptionGroup.user_ids, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment @@ -615,9 +622,10 @@ export const enqueueRssFeedFetch = async ( // Calling the handler function directly. setTimeout(() => { axios - .post(env.queue.rssFeedTaskHandlerUrl, payload, { - headers, - }) + .post( + `${env.queue.rssFeedTaskHandlerUrl}?token=${PUBSUB_VERIFICATION_TOKEN}`, + payload + ) .catch((error) => { logError(error) }) @@ -630,8 +638,7 @@ export const enqueueRssFeedFetch = async ( project: GOOGLE_CLOUD_PROJECT, queue: 'omnivore-rss-queue', payload, - taskHandlerUrl: env.queue.rssFeedTaskHandlerUrl, - requestHeaders: headers, + taskHandlerUrl: `${env.queue.rssFeedTaskHandlerUrl}?token=${PUBSUB_VERIFICATION_TOKEN}`, }) if (!createdTasks || !createdTasks[0].name) { diff --git a/packages/api/test/routers/rss_feed.test.ts b/packages/api/test/routers/rss_feed.test.ts new file mode 100644 index 000000000..a8e35a345 --- /dev/null +++ b/packages/api/test/routers/rss_feed.test.ts @@ -0,0 +1,101 @@ +import chai, { expect } from 'chai' +import 'mocha' +import sinon from 'sinon' +import sinonChai from 'sinon-chai' +import { User } from '../../src/entity/user' +import { SubscriptionType } from '../../src/generated/graphql' +import { createRssSubscriptions } from '../../src/services/subscriptions' +import { deleteUser } from '../../src/services/user' +import * as createTask from '../../src/utils/createTask' +import { createTestUser } from '../db' +import { request } from '../util' + +chai.use(sinonChai) + +describe('Rss feeds Router', () => { + const token = process.env.PUBSUB_VERIFICATION_TOKEN || '' + + let user: User + let user1: User + let user2: User + + before(async () => { + // create test user and login + user = await createTestUser('fakeUser') + await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + + user1 = await createTestUser('fakeUser1') + user2 = await createTestUser('fakeUser2') + // create test subscriptions + const name1 = 'NPR' + const url1 = 'https://www.npr.org/rss/rss.php?id=1001' + const name2 = 'BBC' + const url2 = 'http://feeds.bbci.co.uk/news/rss.xml' + await createRssSubscriptions([ + { + name: name1, + user: { id: user1.id }, + scheduledAt: new Date(), + url: url1, + type: SubscriptionType.Rss, + }, + { + name: name1, + user: { id: user2.id }, + scheduledAt: new Date(), + url: url1, + type: SubscriptionType.Rss, + }, + { + name: name2, + user: { id: user1.id }, + url: url2, + type: SubscriptionType.Rss, + }, + { + name: name2, + user: { id: user2.id }, + // 1 hour in the future + scheduledAt: new Date(Date.now() + 60 * 60 * 1000), + url: url2, + type: SubscriptionType.Rss, + }, + ]) + }) + + after(async () => { + // clean up + await deleteUser(user.id) + await deleteUser(user1.id) + await deleteUser(user2.id) + }) + + it('fetches all scheduled RSS feeds', async () => { + const data = { + message: { + data: Buffer.from('').toString('base64'), + publishTime: new Date().toISOString(), + }, + } + + // fake enqueueRssFeedFetch function + const fake = sinon.replace( + createTask, + 'enqueueRssFeedFetch', + sinon.fake.resolves('task name') + ) + + const res = await request + .post('/svc/pubsub/rss-feed/fetchAll?token=' + token) + .send(data) + .expect(200) + expect(res.text).to.eql('OK') + + // check if enqueueRssFeedFetch is called + expect(fake).to.have.been.called + + sinon.restore() + }) +}) From 1880f2ace4e3a7977bc59a97f9c9c90265f64e60 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 17:25:07 +0800 Subject: [PATCH 67/78] Update rss handler to fetch a group of rss feeds in one task --- packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + .../api/src/resolvers/subscriptions/index.ts | 12 +- packages/api/src/routers/svc/rss_feed.ts | 9 +- packages/api/src/schema.ts | 1 + packages/api/src/utils/createTask.ts | 21 +- packages/rss-handler/src/index.ts | 341 ++++++++++-------- 7 files changed, 225 insertions(+), 161 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 0ad412b4f..6d04536da 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2980,6 +2980,7 @@ export type UpdateSubscriptionInput = { lastFetchedAt?: InputMaybe; lastFetchedChecksum?: InputMaybe; name?: InputMaybe; + scheduledAt?: InputMaybe; status?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index af9064bee..f50844aee 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2393,6 +2393,7 @@ input UpdateSubscriptionInput { lastFetchedAt: Date lastFetchedChecksum: String name: String + scheduledAt: Date status: SubscriptionStatus } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 7bf145452..dd0b8a7c7 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -239,11 +239,12 @@ export const subscribeResolver = authorized< // create a cloud task to fetch rss feed item for the new subscription await enqueueRssFeedFetch({ - user_ids: [uid], + userIds: [uid], url: input.url, - subscription_ids: [newSubscription.id], - scheduled_timestamps: [null], - last_fetched_timestamps: [null], + subscriptionIds: [newSubscription.id], + scheduledDates: [new Date()], // fetch immediately + fetchedDates: [null], + checksums: [null], }) return { @@ -300,6 +301,9 @@ export const updateSubscriptionResolver = authorized< : undefined, lastFetchedChecksum: input.lastFetchedChecksum || undefined, status: input.status || undefined, + scheduledAt: input.scheduledAt + ? new Date(input.scheduledAt) + : undefined, }) return repo.findOneByOrFail({ diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 93077f6e4..053eb7d6f 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -30,10 +30,11 @@ export function rssFeedRouter() { ` SELECT url, - ARRAY_AGG(id) AS subscription_ids, - ARRAY_AGG(user_id) AS user_ids, - ARRAY_AGG(last_fetched_at) AS last_fetched_timestamps, - ARRAY_AGG(scheduled_at) AS scheduled_timestamps + ARRAY_AGG(id) AS "subscriptionIds", + ARRAY_AGG(user_id) AS "userIds", + ARRAY_AGG(last_fetched_at) AS "fetchedDates", + ARRAY_AGG(IFNULL(scheduled_at, NOW())) AS "scheduledDates", + ARRAY_AGG(last_fetched_checksum) AS "checksums" FROM omnivore.subscriptions WHERE diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 2cc4e8b58..4d125b10a 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2550,6 +2550,7 @@ const schema = gql` lastFetchedAt: Date lastFetchedChecksum: String status: SubscriptionStatus + scheduledAt: Date } union UpdateSubscriptionResult = diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 7e09968ea..f6f2ac6dc 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -593,10 +593,11 @@ export const enqueueThumbnailTask = async ( export interface RssSubscriptionGroup { url: string - subscription_ids: string[] - user_ids: string[] - last_fetched_timestamps: (Date | null)[] - scheduled_timestamps: (Date | null)[] + subscriptionIds: string[] + userIds: string[] + fetchedDates: (Date | null)[] + scheduledDates: Date[] + checksums: (string | null)[] } export const enqueueRssFeedFetch = async ( @@ -604,16 +605,16 @@ export const enqueueRssFeedFetch = async ( ): Promise => { const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env const payload = { - subscriptionIds: subscriptionGroup.subscription_ids, + subscriptionIds: subscriptionGroup.subscriptionIds, feedUrl: subscriptionGroup.url, - lastFetchedTimestamps: subscriptionGroup.last_fetched_timestamps.map( + lastFetchedTimestamps: subscriptionGroup.fetchedDates.map( (timestamp) => timestamp?.getTime() || 0 ), // unix timestamp in milliseconds - lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, - scheduledTimestamps: subscriptionGroup.scheduled_timestamps.map( - (timestamp) => timestamp?.getTime() || 0 + lastFetchedChecksums: subscriptionGroup.checksums, + scheduledTimestamps: subscriptionGroup.scheduledDates.map((timestamp) => + timestamp.getTime() ), // unix timestamp in milliseconds - userIds: subscriptionGroup.user_ids, + userIds: subscriptionGroup.userIds, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 8b3021659..2c874f3c1 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -8,10 +8,12 @@ import { promisify } from 'util' import { CONTENT_FETCH_URL, createCloudTask } from './task' interface RssFeedRequest { - subscriptionId: string + subscriptionIds: string[] feedUrl: string - lastFetchedAt: number // unix timestamp in milliseconds - lastFetchedChecksum: string | undefined + lastFetchedTimestamps: number[] // unix timestamp in milliseconds + scheduledTimestamps: number[] // unix timestamp in milliseconds + lastFetchedChecksums: string[] + userIds: string[] } // link can be a string or an object @@ -19,7 +21,12 @@ type RssFeedItemLink = string | { $: { rel?: string; href: string } } function isRssFeedRequest(body: any): body is RssFeedRequest { return ( - 'subscriptionId' in body && 'feedUrl' in body && 'lastFetchedAt' in body + 'subscriptionIds' in body && + 'feedUrl' in body && + 'lastFetchedTimestamps' in body && + 'scheduledTimestamps' in body && + 'userIds' in body && + 'lastFetchedChecksums' in body ) } @@ -53,7 +60,8 @@ const sendUpdateSubscriptionMutation = async ( userId: string, subscriptionId: string, lastFetchedAt: Date, - lastFetchedChecksum: string + lastFetchedChecksum: string, + scheduledAt: Date ) => { const JWT_SECRET = process.env.JWT_SECRET const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT @@ -81,6 +89,7 @@ const sendUpdateSubscriptionMutation = async ( id: subscriptionId, lastFetchedAt, lastFetchedChecksum, + scheduledAt, }, }, }) @@ -155,10 +164,36 @@ const parser = new Parser({ 'updated', 'created', ], - feed: ['dc:date', 'lastBuildDate', 'pubDate'], + feed: [ + 'dc:date', + 'lastBuildDate', + 'pubDate', + 'syn:updatePeriod', + 'syn:updateFrequency', + 'sy:updatePeriod', + 'sy:updateFrequency', + ], }, }) +const getUpdatePeriodInHours = (updatePeriod: string) => { + switch (updatePeriod) { + case 'hourly': + return 1 + case 'daily': + return 24 + case 'weekly': + return 7 * 24 + case 'monthly': + return 30 * 24 + case 'yearly': + return 365 * 24 + default: + // default to hourly + return 1 + } +} + // get link following the order of preference: via, alternate, self const getLink = (links: RssFeedItemLink[]) => { // sort links by preference @@ -185,162 +220,182 @@ const getLink = (links: RssFeedItemLink[]) => { return sortedLinks.find((link) => !!link) } -export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( - async (req, res) => { - if (!process.env.JWT_SECRET) { - console.error('Missing JWT_SECRET in environment') - return res.status(500).send('INTERNAL_SERVER_ERROR') +const processSubscription = async ( + feedUrl: string, + userId: string, + subscriptionId: string, + fetchResult: { content: string; checksum: string }, + lastFetchedAt: number, + scheduledAt: number, + lastFetchedChecksum: string +) => { + let lastItemFetchedAt: Date | null = null + let lastValidItem: Item | null = null + + if (fetchResult.checksum === lastFetchedChecksum) { + console.log('feed has not been updated', feedUrl, lastFetchedChecksum) + return + } + const updatedLastFetchedChecksum = fetchResult.checksum + + // fetch feed + let itemCount = 0 + const feed = await parser.parseString(fetchResult.content) + console.log('Fetched feed', feed.title, new Date()) + + const feedPubDate = (feed['dc:date'] || + feed.pubDate || + feed.lastBuildDate) as string | undefined + console.log('Feed pub date', feedPubDate) + if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { + console.log('Skipping old feed', feedPubDate) + return + } + + // save each item in the feed + for (const item of feed.items) { + // use published or updated if isoDate is not available for atom feeds + item.isoDate = + item.isoDate || + (item.published as string) || + (item.updated as string) || + (item.created as string) + console.log('Processing feed item', item.links, item.isoDate) + + if (!item.links || item.links.length === 0) { + console.log('Invalid feed item', item) + continue } - const token = req.header('Omnivore-Authorization') - if (!token) { - console.error('Missing authorization header') - return res.status(401).send('UNAUTHORIZED') + item.link = getLink(item.links as RssFeedItemLink[]) + if (!item.link) { + console.log('Invalid feed item links', item.links) + continue + } + + console.log('Fetching feed item', item.link) + + const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() + // remember the last valid item + if ( + !lastValidItem || + (lastValidItem.isoDate && publishedAt > new Date(lastValidItem.isoDate)) + ) { + lastValidItem = item + } + + // Max limit per-feed update + if (itemCount > 99) { + continue + } + + // skip old items and items that were published before 24h + if ( + publishedAt < new Date(lastFetchedAt) || + publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) + ) { + console.log('Skipping old feed item', item.link) + continue + } + + const created = await createSavingItemTask(userId, feedUrl, item) + if (!created) { + console.error('Failed to create task for feed item', item.link) + continue + } + + // remember the last item fetched at + if (!lastItemFetchedAt || publishedAt > lastItemFetchedAt) { + lastItemFetchedAt = publishedAt + } + + itemCount = itemCount + 1 + } + + // no items saved + if (!lastItemFetchedAt) { + // the feed has been fetched before, no new valid items found + if (lastFetchedAt || !lastValidItem) { + console.log('No new valid items found') + return + } + + // the feed has never been fetched, save at least the last valid item + const created = await createSavingItemTask(userId, feedUrl, lastValidItem) + if (!created) { + console.error('Failed to create task for feed item', lastValidItem.link) + throw new Error('Failed to create task for feed item') + } + + lastItemFetchedAt = lastValidItem.isoDate + ? new Date(lastValidItem.isoDate) + : new Date() + } + + const updateFrequency = (feed['syn:updateFrequency'] || + feed['sy:updateFrequency'] || + 1) as number + const updatePeriod = (feed['syn:updatePeriod'] || + feed['sy:updatePeriod'] || + 'hourly') as string + const nextScheduledAt = + scheduledAt + + getUpdatePeriodInHours(updatePeriod) * 60 * 60 * 1000 * updateFrequency + + // update subscription lastFetchedAt + const updatedSubscription = await sendUpdateSubscriptionMutation( + userId, + subscriptionId, + lastItemFetchedAt, + updatedLastFetchedChecksum, + new Date(nextScheduledAt) + ) + console.log('Updated subscription', updatedSubscription) +} + +export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { + console.log('query does not include valid token') + return res.sendStatus(403) } try { - let userId: string - - try { - const decoded = jwt.verify(token, process.env.JWT_SECRET) as { - uid: string - } - userId = decoded.uid - } catch (e) { - console.error('Authorization error', e) - return res.status(401).send('UNAUTHORIZED') - } - if (!isRssFeedRequest(req.body)) { console.error('Invalid request body', req.body) return res.status(400).send('INVALID_REQUEST_BODY') } - const { feedUrl, subscriptionId, lastFetchedAt, lastFetchedChecksum } = - req.body - console.log('Processing feed', feedUrl, lastFetchedAt) - - let lastItemFetchedAt: Date | null = null - let lastValidItem: Item | null = null + const { + feedUrl, + subscriptionIds, + lastFetchedTimestamps, + scheduledTimestamps, + userIds, + lastFetchedChecksums, + } = req.body + console.log('Processing feed', feedUrl) const fetchResult = await fetchAndChecksum(feedUrl) - if (fetchResult.checksum === lastFetchedChecksum) { - console.log('feed has not been updated', feedUrl, lastFetchedChecksum) - return res.send('ok') - } - const updatedLastFetchedChecksum = fetchResult.checksum - // fetch feed - let itemCount = 0 - const feed = await parser.parseString(fetchResult.content) - console.log('Fetched feed', feed.title, new Date()) + for (let i = 0; i < subscriptionIds.length; i++) { + const subscriptionId = subscriptionIds[i] + const lastFetchedAt = lastFetchedTimestamps[i] + const scheduledAt = scheduledTimestamps[i] + const userId = userIds[i] + const lastFetchedChecksum = lastFetchedChecksums[i] - const feedPubDate = (feed['dc:date'] || - feed.pubDate || - feed.lastBuildDate) as string | undefined - console.log('Feed pub date', feedPubDate) - if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { - console.log('Skipping old feed', feedPubDate) - return res.send('ok') - } - - // save each item in the feed - for (const item of feed.items) { - // use published or updated if isoDate is not available for atom feeds - item.isoDate = - item.isoDate || - (item.published as string) || - (item.updated as string) || - (item.created as string) - console.log('Processing feed item', item.links, item.isoDate) - - if (!item.links || item.links.length === 0) { - console.log('Invalid feed item', item) - continue - } - - item.link = getLink(item.links as RssFeedItemLink[]) - if (!item.link) { - console.log('Invalid feed item links', item.links) - continue - } - - console.log('Fetching feed item', item.link) - - const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() - // remember the last valid item - if ( - !lastValidItem || - (lastValidItem.isoDate && - publishedAt > new Date(lastValidItem.isoDate)) - ) { - lastValidItem = item - } - - // Max limit per-feed update - if (itemCount > 99) { - continue - } - - // skip old items and items that were published before 24h - if ( - publishedAt < new Date(lastFetchedAt) || - publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) - ) { - console.log('Skipping old feed item', item.link) - continue - } - - const created = await createSavingItemTask(userId, feedUrl, item) - if (!created) { - console.error('Failed to create task for feed item', item.link) - continue - } - - // remember the last item fetched at - if (!lastItemFetchedAt || publishedAt > lastItemFetchedAt) { - lastItemFetchedAt = publishedAt - } - - itemCount = itemCount + 1 - } - - // no items saved - if (!lastItemFetchedAt) { - // the feed has been fetched before, no new valid items found - if (lastFetchedAt || !lastValidItem) { - console.log('No new valid items found') - return res.send('ok') - } - - // the feed has never been fetched, save at least the last valid item - const created = await createSavingItemTask( + await processSubscription( + subscriptionId, userId, feedUrl, - lastValidItem + fetchResult, + lastFetchedAt, + scheduledAt, + lastFetchedChecksum ) - if (!created) { - console.error( - 'Failed to create task for feed item', - lastValidItem.link - ) - return res.status(500).send('INTERNAL_SERVER_ERROR') - } - - lastItemFetchedAt = lastValidItem.isoDate - ? new Date(lastValidItem.isoDate) - : new Date() } - // update subscription lastFetchedAt - const updatedSubscription = await sendUpdateSubscriptionMutation( - userId, - subscriptionId, - lastItemFetchedAt, - updatedLastFetchedChecksum - ) - console.log('Updated subscription', updatedSubscription) - res.send('ok') } catch (e) { console.error('Error while parsing RSS feed', e) From 750ccb0c42061aeee89793f90bab93afab1cb43b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 17:44:53 +0800 Subject: [PATCH 68/78] fix conflicts with main --- packages/api/src/routers/svc/rss_feed.ts | 6 ++---- ...ion.sql => 0139.do.add_scheduled_at_to_subscription.sql} | 0 ...n.sql => 0139.undo.add_scheduled_at_to_subscription.sql} | 0 3 files changed, 2 insertions(+), 4 deletions(-) rename packages/db/migrations/{0138.do.add_scheduled_at_to_subscription.sql => 0139.do.add_scheduled_at_to_subscription.sql} (100%) rename packages/db/migrations/{0138.undo.add_scheduled_at_to_subscription.sql => 0139.undo.add_scheduled_at_to_subscription.sql} (100%) diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 053eb7d6f..3b6a21247 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -33,8 +33,8 @@ export function rssFeedRouter() { ARRAY_AGG(id) AS "subscriptionIds", ARRAY_AGG(user_id) AS "userIds", ARRAY_AGG(last_fetched_at) AS "fetchedDates", - ARRAY_AGG(IFNULL(scheduled_at, NOW())) AS "scheduledDates", - ARRAY_AGG(last_fetched_checksum) AS "checksums" + ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", + ARRAY_AGG(last_fetched_checksum) AS checksums FROM omnivore.subscriptions WHERE @@ -47,8 +47,6 @@ export function rssFeedRouter() { [SubscriptionType.Rss, SubscriptionStatus.Active] )) as RssSubscriptionGroup[] - logger.info('scheduledSubscriptions', subscriptionGroups) - // create a cloud taks to fetch rss feed item for each subscription await Promise.all( subscriptionGroups.map((subscriptionGroup) => { diff --git a/packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0139.do.add_scheduled_at_to_subscription.sql similarity index 100% rename from packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql rename to packages/db/migrations/0139.do.add_scheduled_at_to_subscription.sql diff --git a/packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0139.undo.add_scheduled_at_to_subscription.sql similarity index 100% rename from packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql rename to packages/db/migrations/0139.undo.add_scheduled_at_to_subscription.sql From 25c0ee3b340656fcb4bbb2520c62b43d42fc11f7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 21:08:54 +0800 Subject: [PATCH 69/78] return 1 for updateFrequency if it is not a number --- packages/rss-handler/src/index.ts | 34 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2c874f3c1..28a87bd3d 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -176,7 +176,27 @@ const parser = new Parser({ }, }) -const getUpdatePeriodInHours = (updatePeriod: string) => { +const getUpdateFrequency = (feed: any) => { + const updateFrequency = (feed['syn:updateFrequency'] || + feed['sy:updateFrequency']) as string | undefined + + if (!updateFrequency) { + return 1 + } + + const frequency = parseInt(updateFrequency, 10) + if (isNaN(frequency)) { + return 1 + } + + return frequency +} + +const getUpdatePeriodInHours = (feed: any) => { + const updatePeriod = (feed['syn:updatePeriod'] || feed['sy:updatePeriod']) as + | string + | undefined + switch (updatePeriod) { case 'hourly': return 1 @@ -332,15 +352,9 @@ const processSubscription = async ( : new Date() } - const updateFrequency = (feed['syn:updateFrequency'] || - feed['sy:updateFrequency'] || - 1) as number - const updatePeriod = (feed['syn:updatePeriod'] || - feed['sy:updatePeriod'] || - 'hourly') as string - const nextScheduledAt = - scheduledAt + - getUpdatePeriodInHours(updatePeriod) * 60 * 60 * 1000 * updateFrequency + const updateFrequency = getUpdateFrequency(feed) + const updatePeriodInMs = getUpdatePeriodInHours(feed) * 60 * 60 * 1000 + const nextScheduledAt = scheduledAt + updatePeriodInMs * updateFrequency // update subscription lastFetchedAt const updatedSubscription = await sendUpdateSubscriptionMutation( From f750648824651a2088402985139762766c73d31a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 21:46:43 +0800 Subject: [PATCH 70/78] fix importer triggers thumbnailer unexpectedly --- packages/puppeteer-parse/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b3413a244..b877b8eec 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -272,7 +272,7 @@ const sendSavePageMutation = async (userId, input) => { } }`, variables: { - input: Object.assign({}, input , { source: 'puppeteer-parse' }), + input: Object.assign({}, input), }, }); @@ -341,7 +341,7 @@ async function fetchContent(req, res) { const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); const state = req.body.state const labels = req.body.labels - const source = req.body.source || 'parseContent'; + const source = req.body.source || 'puppeteer-parse'; const taskId = req.body.taskId; // taskId is used to update import status const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); const locale = (req.query ? req.query.locale : undefined) || (req.body ? req.body.locale : undefined); @@ -473,6 +473,7 @@ async function fetchContent(req, res) { rssFeedUrl, savedAt, publishedAt, + source, }); if (!apiResponse) { logRecord.error = 'error while saving page'; From d746510358ec2cbbca0855f4d3bbabaf63f4f444 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 21:50:16 +0800 Subject: [PATCH 71/78] cont --- packages/puppeteer-parse/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b877b8eec..78cdc87dd 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -272,7 +272,7 @@ const sendSavePageMutation = async (userId, input) => { } }`, variables: { - input: Object.assign({}, input), + input, }, }); From efb58f860400725970a7e2ec04ece04f1ae8e1f3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 10:06:28 +0800 Subject: [PATCH 72/78] avoid using string interop in graphql --- .../api/test/resolvers/subscriptions.test.ts | 24 +++++++------------ packages/api/test/util.ts | 9 ++++--- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 22ee107b1..085845eb4 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -350,17 +350,9 @@ describe('Subscriptions API', () => { }) describe('Subscribe API', () => { - const query = ( - name: string | null, - url: string | null, - subscriptionType: SubscriptionType | null - ) => ` - mutation { - subscribe(input: { - name: ${name ? `"${name}"` : null} - url: ${url ? `"${url}"` : null} - subscriptionType: ${subscriptionType} - }) { + const query = ` + mutation Subscribe($input: SubscribeInput!){ + subscribe(input: $input) { ... on SubscribeSuccess { subscriptions { id @@ -410,8 +402,9 @@ describe('Subscriptions API', () => { it('returns an error', async () => { const res = await graphqlRequest( - query(null, url, subscriptionType), - authToken + query, + authToken, + { input: { url, subscriptionType } }, ).expect(200) expect(res.body.data.subscribe.errorCodes).to.eql([ 'ALREADY_SUBSCRIBED', @@ -421,8 +414,9 @@ describe('Subscriptions API', () => { it('creates a rss subscription', async () => { const res = await graphqlRequest( - query(null, url, subscriptionType), - authToken + query, + authToken, + { input: { url, subscriptionType } }, ).expect(200) expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') diff --git a/packages/api/test/util.ts b/packages/api/test/util.ts index 02bac0f99..8268c1503 100644 --- a/packages/api/test/util.ts +++ b/packages/api/test/util.ts @@ -17,15 +17,14 @@ export const stopApolloServer = async () => { export const graphqlRequest = ( query: string, - authToken?: string + authToken: string, + variables?: Record, ): supertest.Test => { return request .post(apollo.graphqlPath) - .send({ - query, - }) + .send({ query, variables }) .set('Accept', 'application/json') - .set('authorization', authToken || '') + .set('authorization', authToken) .expect('Content-Type', /json/) } From 4e36a5809ea75a0a68aa0f1eb2c93177c2082c00 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 11:05:58 +0800 Subject: [PATCH 73/78] fix failure of re-subscribing rss feed --- packages/api/src/generated/graphql.ts | 3 +- packages/api/src/generated/schema.graphql | 3 +- .../api/src/resolvers/subscriptions/index.ts | 116 ++++++++++-------- packages/api/src/schema.ts | 3 +- packages/api/src/services/subscriptions.ts | 11 +- .../api/test/resolvers/subscriptions.test.ts | 43 +++++-- 6 files changed, 100 insertions(+), 79 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 6d04536da..3bbc9a88d 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2616,9 +2616,8 @@ export enum SubscribeErrorCode { } export type SubscribeInput = { - name?: InputMaybe; subscriptionType?: InputMaybe; - url?: InputMaybe; + url: Scalars['String']; }; export type SubscribeResult = SubscribeError | SubscribeSuccess; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index f50844aee..e5192b308 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2058,9 +2058,8 @@ enum SubscribeErrorCode { } input SubscribeInput { - name: String subscriptionType: SubscriptionType - url: String + url: String! } union SubscribeResult = SubscribeError | SubscribeSuccess diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index dd0b8a7c7..8d5b74a83 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -175,25 +175,8 @@ export const subscribeResolver = authorized< SubscribeSuccessPartial, SubscribeError, MutationSubscribeArgs ->(async (_, { input }, { authTrx, uid, log }) => { - log.info('subscribeResolver') - +>(async (_, { input }, { uid, log }) => { try { - // find existing subscription - const subscription = await authTrx((t) => - t.getRepository(Subscription).findOneBy({ - url: input.url || undefined, - name: input.name || undefined, - user: { id: uid }, - type: input.subscriptionType || SubscriptionType.Rss, // default to rss - }) - ) - if (subscription) { - return { - errorCodes: [SubscribeErrorCode.AlreadySubscribed], - } - } - analytics.track({ userId: uid, event: 'subscribed', @@ -203,58 +186,83 @@ export const subscribeResolver = authorized< }, }) - // create new rss subscription - if (input.url) { - const MAX_RSS_SUBSCRIPTIONS = 150 - // validate rss feed - const feed = await parser.parseURL(input.url) - - // limit number of rss subscriptions to 50 - const newSubscriptions = (await authTrx((t) => - t.query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) - select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions - where user_id = $5 and type = 'RSS' and status = 'ACTIVE' - having count(*) < $7 - returning *;`, - [ - feed.title, - input.url, - feed.description || null, - SubscriptionType.Rss, - uid, - feed.image?.url || null, - MAX_RSS_SUBSCRIPTIONS, - ] - ) - )) as Subscription[] - - if (newSubscriptions.length === 0) { + // find existing subscription + const existingSubscription = await getRepository(Subscription).findOneBy({ + url: input.url, + user: { id: uid }, + type: SubscriptionType.Rss, + }) + if (existingSubscription) { + if (existingSubscription.status === SubscriptionStatus.Active) { return { - errorCodes: [SubscribeErrorCode.ExceededMaxSubscriptions], + errorCodes: [SubscribeErrorCode.AlreadySubscribed], } } - const newSubscription = newSubscriptions[0] + // re-subscribe + const updatedSubscription = await getRepository(Subscription).save({ + ...existingSubscription, + status: SubscriptionStatus.Active, + }) - // create a cloud task to fetch rss feed item for the new subscription + // create a cloud task to fetch rss feed item for resub subscription await enqueueRssFeedFetch({ userIds: [uid], url: input.url, - subscriptionIds: [newSubscription.id], + subscriptionIds: [updatedSubscription.id], scheduledDates: [new Date()], // fetch immediately - fetchedDates: [null], - checksums: [null], + fetchedDates: [updatedSubscription.lastFetchedAt || null], + checksums: [updatedSubscription.lastFetchedChecksum || null], }) return { - subscriptions: [newSubscription], + subscriptions: [updatedSubscription], } } - log.info('missing url or name') + // create new rss subscription + const MAX_RSS_SUBSCRIPTIONS = 150 + // validate rss feed + const feed = await parser.parseURL(input.url) + + // limit number of rss subscriptions to 150 + const results = (await getRepository(Subscription).query( + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) + select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions + where user_id = $5 and type = 'RSS' and status = 'ACTIVE' + having count(*) < $7 + returning *;`, + [ + feed.title, + input.url, + feed.description || null, + SubscriptionType.Rss, + uid, + feed.image?.url || null, + MAX_RSS_SUBSCRIPTIONS, + ] + )) as Subscription[] + + if (results.length === 0) { + return { + errorCodes: [SubscribeErrorCode.ExceededMaxSubscriptions], + } + } + + const newSubscription = results[0] + + // create a cloud task to fetch rss feed item for the new subscription + await enqueueRssFeedFetch({ + userIds: [uid], + url: input.url, + subscriptionIds: [newSubscription.id], + scheduledDates: [new Date()], // fetch immediately + fetchedDates: [null], + checksums: [null], + }) + return { - errorCodes: [SubscribeErrorCode.BadRequest], + subscriptions: [newSubscription], } } catch (error) { log.error('failed to subscribe', error) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4d125b10a..de3f5209c 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2538,8 +2538,7 @@ const schema = gql` } input SubscribeInput { - url: String - name: String + url: String! subscriptionType: SubscriptionType } diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 9e6e78b37..fcf25af01 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -205,15 +205,8 @@ export const createSubscription = async ( }) } -export const deleteSubscription = async ( - id: string, - userId: string -): Promise => { - return authTrx( - (tx) => tx.getRepository(Subscription).delete(id), - undefined, - userId - ) +export const deleteSubscription = async (id: string): Promise => { + return getRepository(Subscription).delete(id) } export const createRssSubscriptions = async ( diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 085845eb4..02f7cd1b8 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -397,21 +397,47 @@ describe('Subscriptions API', () => { }) after(async () => { - await deleteSubscription(existingSubscription.id, user.id) + await deleteSubscription(existingSubscription.id) }) it('returns an error', async () => { - const res = await graphqlRequest( - query, - authToken, - { input: { url, subscriptionType } }, - ).expect(200) + const res = await graphqlRequest(query, authToken, { + input: { url, subscriptionType }, + }).expect(200) expect(res.body.data.subscribe.errorCodes).to.eql([ 'ALREADY_SUBSCRIBED', ]) }) }) + context('when the user unsubscribed the feed', () => { + let existingSubscription: Subscription + + before(async () => { + existingSubscription = await createSubscription( + user.id, + 'RSS Feed', + undefined, + SubscriptionStatus.Unsubscribed, + url, + subscriptionType, + url + ) + }) + + after(async () => { + await deleteSubscription(existingSubscription.id) + }) + + it('re-subscribes the user', async () => { + const res = await graphqlRequest(query, authToken, { + input: { url, subscriptionType }, + }).expect(200) + expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) + expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') + }) + }) + it('creates a rss subscription', async () => { const res = await graphqlRequest( query, @@ -422,10 +448,7 @@ describe('Subscriptions API', () => { expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') // clean up - await deleteSubscription( - res.body.data.subscribe.subscriptions[0].id, - user.id - ) + await deleteSubscription(res.body.data.subscribe.subscriptions[0].id) }) }) }) From b9e2f9ee53f0bf53aff2880f542bb01e51f52fe5 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 20 Oct 2023 11:00:31 +0800 Subject: [PATCH 74/78] Add a longer rate limit window on createaccount/reset password, reduce api rate limit hits --- packages/api/src/routers/auth/auth_router.ts | 12 ++++++++++++ packages/api/src/server.ts | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 333391d62..3b38048a3 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -46,6 +46,7 @@ import { } from './google_auth' import { createWebAuthToken } from './jwt_helpers' import { createMobileAccountCreationResponse } from './mobile/account_creation' +import rateLimit from 'express-rate-limit' export interface SignupRequest { email: string @@ -80,6 +81,14 @@ export const isValidSignupRequest = (obj: any): obj is SignupRequest => { ) } +// The hourly limiter is used on the create account, +// and reset password endpoints +// this limits users to five operations per an hour +const hourlyLimiter = rateLimit({ + windowMs: 60 * 60 * 1000, + max: 5, +}) + export function authRouter() { const router = express.Router() @@ -108,6 +117,7 @@ export function authRouter() { ) router.post( '/create-account', + hourlyLimiter, cors(corsConfig), async (req, res) => { const { name, bio, username } = req.body @@ -480,6 +490,7 @@ export function authRouter() { router.post( '/email-signup', + hourlyLimiter, cors(corsConfig), async (req: express.Request, res: express.Response) => { if (!isValidSignupRequest(req.body)) { @@ -599,6 +610,7 @@ export function authRouter() { router.post( '/forgot-password', + hourlyLimiter, cors(corsConfig), async (req: express.Request, res: express.Response) => { const email = req.body.email?.trim() as string // trim whitespace diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 9e41af64d..b40a82bb8 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -68,7 +68,7 @@ export const createApp = (): { const token = getTokenByRequest(req) try { const claims = await getClaimsByToken(token) - return claims ? 100 : 15 + return claims ? 60 : 15 } catch (e) { console.log('non-authenticated request') return 15 From a23b67b70b8e7315ca09b60f5d94f8886fe11378 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 20 Oct 2023 11:44:52 +0800 Subject: [PATCH 75/78] Dont rate limit auth in local env so tests can run --- packages/api/src/routers/auth/auth_router.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 3b38048a3..7779b9af8 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -87,6 +87,7 @@ export const isValidSignupRequest = (obj: any): obj is SignupRequest => { const hourlyLimiter = rateLimit({ windowMs: 60 * 60 * 1000, max: 5, + skip: (req) => env.dev.isLocal, }) export function authRouter() { From 23eb7ea76fdc83d569f57785254aeb5ec5cbc265 Mon Sep 17 00:00:00 2001 From: Rudra Date: Fri, 20 Oct 2023 11:51:02 +0530 Subject: [PATCH 76/78] fix typo --- packages/api/src/resolvers/reaction/index.ts | 2 +- packages/api/src/routers/svc/content.ts | 2 +- packages/pdf-handler/src/pdf.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/src/resolvers/reaction/index.ts b/packages/api/src/resolvers/reaction/index.ts index aec16962a..ceea66324 100644 --- a/packages/api/src/resolvers/reaction/index.ts +++ b/packages/api/src/resolvers/reaction/index.ts @@ -17,7 +17,7 @@ export type PartialCreateReactionSuccess = Merge< // if ((!userArticleId && !highlightId) || (userArticleId && highlightId)) { // // One reaction target is required -// // Higlight replies hasn't supported yet +// // Highlight replies hasn't supported yet // return { // errorCodes: [CreateReactionErrorCode.BadTarget], // } diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index ce184ea5b..70723429b 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -91,7 +91,7 @@ export function contentServiceRouter() { if (msg.description) itemToUpdate.description = msg.description // This event is fired after the file is fully uploaded, - // so along with updateing content, we mark it as + // so along with updating content, we mark it as // succeeded. itemToUpdate.state = LibraryItemState.Succeeded diff --git a/packages/pdf-handler/src/pdf.ts b/packages/pdf-handler/src/pdf.ts index 48d85b637..0585b742b 100644 --- a/packages/pdf-handler/src/pdf.ts +++ b/packages/pdf-handler/src/pdf.ts @@ -14,7 +14,7 @@ interface Page { lines: string[] } -// Unused at the moment -- comented out for now to satisfy linter +// Unused at the moment -- commented out for now to satisfy linter const MAX_TITLE_LENGTH = 95 type MetadataInfoKey = From 6618a6ed19af4cdf679976d6ba69445b9b83825b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 14:31:16 +0800 Subject: [PATCH 77/78] migrate popular reads into database --- packages/api/src/repository/library_item.ts | 42 + .../api/src/resolvers/popular_reads/index.ts | 6 +- packages/api/src/services/popular_reads.ts | 192 +- .../popular_reads/elad_meetings-content.html | 36 - .../popular_reads/elad_meetings-original.html | 1497 --- .../jonbo_digital_tools-content.html | 208 - .../jonbo_digital_tools-original.html | 453 - .../omnivore_get_started-content.html | 331 - .../omnivore_get_started-original.html | 1670 --- .../popular_reads/omnivore_ios-content.html | 94 - .../popular_reads/omnivore_ios-original.html | 1456 --- .../omnivore_organize-content.html | 73 - .../omnivore_organize-original.html | 1022 -- .../power_read_it_later-content.html | 239 - .../power_read_it_later-original.html | 1551 --- .../popular_reads/rlove_carnitas-content.html | 34 - .../rlove_carnitas-original.html | 1230 -- .../db/migrations/0140.do.popular_read.sql | 9931 +++++++++++++++++ .../db/migrations/0140.undo.popular_read.sql | 9 + 19 files changed, 10007 insertions(+), 10067 deletions(-) delete mode 100644 packages/api/src/services/popular_reads/elad_meetings-content.html delete mode 100644 packages/api/src/services/popular_reads/elad_meetings-original.html delete mode 100644 packages/api/src/services/popular_reads/jonbo_digital_tools-content.html delete mode 100644 packages/api/src/services/popular_reads/jonbo_digital_tools-original.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_get_started-content.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_get_started-original.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_ios-content.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_ios-original.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_organize-content.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_organize-original.html delete mode 100644 packages/api/src/services/popular_reads/power_read_it_later-content.html delete mode 100644 packages/api/src/services/popular_reads/power_read_it_later-original.html delete mode 100644 packages/api/src/services/popular_reads/rlove_carnitas-content.html delete mode 100644 packages/api/src/services/popular_reads/rlove_carnitas-original.html create mode 100755 packages/db/migrations/0140.do.popular_read.sql create mode 100755 packages/db/migrations/0140.undo.popular_read.sql diff --git a/packages/api/src/repository/library_item.ts b/packages/api/src/repository/library_item.ts index 1a428d65b..1e7e3d708 100644 --- a/packages/api/src/repository/library_item.ts +++ b/packages/api/src/repository/library_item.ts @@ -17,4 +17,46 @@ export const libraryItemRepository = appDataSource countByCreatedAt(createdAt: Date) { return this.countBy({ createdAt }) }, + + createByPopularRead(name: string, userId: string) { + return this.query( + ` + INSERT INTO omnivore.library_item ( + slug, + readable_content, + original_content, + description, + title, + author, + original_url, + item_type, + thumbnail, + published_at, + site_name, + user_id, + word_count + ) + SELECT + slug, + readable_content, + original_content, + description, + title, + author, + original_url, + $1, + thumbnail, + published_at, + site_name, + $2, + word_count + FROM + omnivore.popular_read + WHERE + key = $3 + RETURNING * + `, + ['ARTICLE', userId, name] + ) as Promise + }, }) diff --git a/packages/api/src/resolvers/popular_reads/index.ts b/packages/api/src/resolvers/popular_reads/index.ts index 3f91fe2c9..30e1de08c 100644 --- a/packages/api/src/resolvers/popular_reads/index.ts +++ b/packages/api/src/resolvers/popular_reads/index.ts @@ -11,12 +11,12 @@ export const addPopularReadResolver = authorized< AddPopularReadError, MutationAddPopularReadArgs >(async (_, { name }, { uid }) => { - const item = await addPopularRead(uid, name) - if (!item) { + const items = await addPopularRead(uid, name) + if (items.length === 0) { return { errorCodes: [AddPopularReadErrorCode.NotFound] } } return { - pageId: item.id, + pageId: items[0].id, } }) diff --git a/packages/api/src/services/popular_reads.ts b/packages/api/src/services/popular_reads.ts index c98fd9851..de6daab4d 100644 --- a/packages/api/src/services/popular_reads.ts +++ b/packages/api/src/services/popular_reads.ts @@ -1,93 +1,23 @@ import * as httpContext from 'express-http-context2' -import { readFileSync } from 'fs' -import path from 'path' -import { DeepPartial, EntityManager } from 'typeorm' +import { EntityManager } from 'typeorm' import { appDataSource } from '../data_source' -import { LibraryItem } from '../entity/library_item' -import { PageType } from '../generated/graphql' import { authTrx } from '../repository' import { libraryItemRepository } from '../repository/library_item' -import { generateSlug, stringToHash, wordsCount } from '../utils/helpers' import { logger } from '../utils/logger' -import { createLibraryItem } from './library_item' -type PopularRead = { - url: string - title: string - author: string - description: string - previewImage?: string - publishedAt: Date - siteName: string - - content: string - originalHtml: string -} - -const popularRead = (key: string): PopularRead | undefined => { - const metadata = popularReads.find((pr) => pr.key === key) - if (!metadata) { - return undefined - } - - try { - const content = readFileSync( - path.resolve(__dirname, `popular_reads/${key}-content.html`), - 'utf8' - ) - const originalHtml = readFileSync( - path.resolve(__dirname, `./popular_reads/${key}-original.html`), - 'utf8' - ) - if (!content || !originalHtml) { - return undefined - } - - return { - ...metadata, - content, - originalHtml, - } - } catch (e) { - logger.info('error adding popular read', e) - return undefined - } -} - -const popularReadToLibraryItem = ( +export const addPopularRead = async ( + userId: string, name: string, - userId: string -): DeepPartial | null => { - const pr = popularRead(name) - if (!pr) { - return null - } - - return { - slug: generateSlug(pr.title), - readableContent: pr.content, - originalContent: pr.originalHtml, - description: pr.description, - title: pr.title, - author: pr.author, - originalUrl: pr.url, - itemType: PageType.Article, - textContentHash: stringToHash(pr.content), - thumbnail: pr.previewImage, - publishedAt: pr.publishedAt, - siteName: pr.siteName, - user: { id: userId }, - wordCount: wordsCount(pr.content), - } -} - -export const addPopularRead = async (userId: string, name: string) => { - const itemToSave = popularReadToLibraryItem(name, userId) - if (!itemToSave) { - return null - } - - return createLibraryItem(itemToSave, userId) + entityManager?: EntityManager +) => { + return authTrx( + async (tx) => + tx + .withRepository(libraryItemRepository) + .createByPopularRead(name, userId), + entityManager, + userId + ) } const addPopularReads = async ( @@ -95,15 +25,15 @@ const addPopularReads = async ( userId: string, entityManager: EntityManager ) => { - const libraryItems = names - .map((name) => popularReadToLibraryItem(name, userId)) - .filter((pr) => pr !== null) as DeepPartial[] - - return authTrx( - async (tx) => tx.withRepository(libraryItemRepository).save(libraryItems), - entityManager, - userId - ) + // insert one by one to ensure that the order is preserved + for (const name of names) { + try { + await addPopularRead(userId, name, entityManager) + } catch (error) { + logger.error('failed to add popular read', error) + continue + } + } } export const addPopularReadsForNewUser = async ( @@ -132,81 +62,3 @@ export const addPopularReadsForNewUser = async ( defaultReads.push('omnivore_get_started') await addPopularReads(defaultReads, userId, em) } - -const popularReads = [ - { - key: 'omnivore_get_started', - url: 'https://blog.omnivore.app/p/getting-started-with-omnivore', - title: 'Getting Started with Omnivore', - author: 'The Omnivore Team', - description: 'Get the most out of Omnivore by learning how to use it.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sxQnqya1QNApB7ZAGPj9K20AU6sw0UAnjmAIy2ub8hUU/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F658efff4-341a-4720-8cf6-9b2bdbedfaa7_800x668.gif', - publishedAt: new Date('2021-10-13'), - siteName: 'Omnivore Blog', - }, - { - key: 'omnivore_ios', - url: 'https://blog.omnivore.app/p/saving-links-from-your-iphone-or', - title: 'Saving Links from Your iPhone or iPad', - author: 'Omnivore', - description: 'Learn how to save articles on iOS.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sWDfv7sARTIdAlx6Rw_6t-QwL3T9aniEJRa1-jVaglNg/https://substackcdn.com/image/youtube/w_728,c_limit/k6RkIqepAig', - publishedAt: new Date('2021-10-19'), - siteName: 'Omnivore Blog', - }, - { - key: 'omnivore_organize', - url: 'https://blog.omnivore.app/p/organize-your-omnivore-library-with', - title: 'Organize your Omnivore library with labels', - author: 'The Omnivore Team', - description: 'Use labels to organize your Omnivore library.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sTgJ5Q0XIg_EHdmPWcxtXFmkjn8T6hkJt7S9ziClagYo/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fdaf07af7-5cdb-4ecc-aace-1a46de3e9c58_1827x1090.png', - publishedAt: new Date('2022-04-18'), - siteName: 'Omnivore Blog', - }, - { - key: 'rlove_carnitas', - url: 'https://medium.com/@rlove/carnitas-ff0ef1044ae9', - title: 'Slow-Braised Carnitas Recipe', - author: 'Robert Love', - description: - 'Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. This is not an authentic recipe.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/88x88,sIcDXt3Ar0baKG1e1Yi1e2VUZFL85xPlOeEfAxF-s-Nw/https://miro.medium.com/max/1200/1*Wl-dMBJpSgPUxUOnPQthyg.jpeg', - publishedAt: new Date('2017-02-24'), - siteName: '@rlove', - }, - { - key: 'power_read_it_later', - url: 'https://fortelabs.co/blog/the-secret-power-of-read-it-later-apps', - title: 'The Secret Power of ‘Read It Later’ Apps', - author: 'Tiago Forte', - description: - 'At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sGN5R34M5z068QMXDZD32CQD6mCbxc47hWXm__JVUePE/https://fortelabs.com/wp-content/uploads/2015/11/1rPXwIczUJRCE54v8FfAHGw.jpeg', - publishedAt: new Date('2022-01-24'), - siteName: 'Forte Labs', - }, - { - key: 'elad_meetings', - url: 'http://blog.eladgil.com/2018/07/meeting-etiquette.html', - title: 'Better Meetings', - author: 'Elad Gil', - description: 'How to make meetings more productive.', - publishedAt: new Date('2018-07-02'), - siteName: 'Elad Blog', - }, - { - key: 'jonbo_digital_tools', - url: 'https://jon.bo/posts/digital-tools/', - title: 'Digital Tools I Wish Existed', - author: 'Jonathan Borichevskiy', - description: `My digital life in a nutshell: I discover relevant content I don’t have time to consume...`, - publishedAt: new Date('2019-11-28'), - siteName: 'JON.BO', - }, -] diff --git a/packages/api/src/services/popular_reads/elad_meetings-content.html b/packages/api/src/services/popular_reads/elad_meetings-content.html deleted file mode 100644 index 34f8a7aee..000000000 --- a/packages/api/src/services/popular_reads/elad_meetings-content.html +++ /dev/null @@ -1,36 +0,0 @@ -
-
-

As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
-
1. Determine who is necessary in the meeting.
- Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
-
2. Send out an agenda in advance.
- What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
-
Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
-
3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
- If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
-
If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
-
4. Kick off the meeting with objectives.
- Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
-
5. Assign a note taker.
- Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
-
6. Send out meeting notes.
- Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
-
Meeting notes would optimally include:
- a. Subject/topic of meeting
- b. Date
- c. Attendees
- d. Actions/decisions
- e. Agenda
- f. Detailed notes
-
7. Clean up the meeting calendar ongoing.

- Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
-
NOTES

- [1] There may be legal reasons to not take notes in specific meetings. Consult with the company's GC or external counsel on training for meeting note etiquette. -

-

MY BOOK -

-

You can pre-order the High Growth Handbook here.
-
-

-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/elad_meetings-original.html b/packages/api/src/services/popular_reads/elad_meetings-original.html deleted file mode 100644 index 054b6f5f9..000000000 --- a/packages/api/src/services/popular_reads/elad_meetings-original.html +++ /dev/null @@ -1,1497 +0,0 @@ - - - - - - - - - - - - - - - - - - Elad Blog: Better Meetings - - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- Monday, July 2, 2018 -

-
-
-
- - -

- Better Meetings -

-
-
-
-
-
- As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
-
1. Determine who is necessary in the meeting.
- Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
-
2. Send out an agenda in advance.
- What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
-
Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
-
3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
- If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
-
If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
-
4. Kick off the meeting with objectives.
- Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
-
5. Assign a note taker.
- Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
-
6. Send out meeting notes.
- Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
-
Meeting notes would optimally include:
- a. Subject/topic of meeting
- b. Date
- c. Attendees
- d. Actions/decisions
- e. Agenda
- f. Detailed notes
-
7. Clean up the meeting calendar ongoing.

- Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
-
NOTES

- [1] There may be legal reasons to not take notes in specific meetings. Consult with the company's GC or external counsel on training for meeting note etiquette.
-
-
-
-
- MY BOOK -
You can pre-order the High Growth Handbook here.
-

-
-
-
-
- RELATED POSTS -
- - - - - - -
-
-
- -
-
- -
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
- - - - - - diff --git a/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html b/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html deleted file mode 100644 index 711fe33ef..000000000 --- a/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html +++ /dev/null @@ -1,208 +0,0 @@ -
-
-

My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later.

-

There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed:

-
    -
  • -

    The more seamless the acquisition & ingestion, the more engaged I am with the content

    -
  • -
  • -

    Insights are just as likely to be found in a 400-page book as in a 40-minute podcast

    -
  • -
  • -

    Notes and their subsequent review are essential for long-term retention

    -
  • -
  • -

    Recommendations from other humans are as good, if not better, than algorithmic suggestions

    -
  • -
-

In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here.

-

Queue management for inbound digital content # -

-

Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance:

-
    -
  • -

    Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more.

    -
  • -
  • -

    Every book, article, post, or tweet has the potential to lead to more content.

    -
  • -
  • -

    Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs.

    -
  • -
  • -

    I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee?

    -
  • -
  • -

    Learning, work, news, and entertainment all have different priorities in my life (roughly in that order).

    -
  • -
  • -

    I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month.

    -
  • -
  • -

    I’m not always connected to a stable internet connection.

    -
  • -
  • -

    If it’s a long piece of content I want my position saved reliably so I can resume at a later point.

    -
  • -
  • -

    I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation.

    -
  • -
  • -

    I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place.

    -
  • -
  • -

    I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat.

    -
  • -
-
- -
Relevant XKCD, as is tradition
-
-

Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one.

-

Honorable Mentions: Pocket, Instapaper -

-

A universal book log, recommendation & sharing system # -

-

I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof.

-

Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry.

-

Whatever this tool for managing my ever-growing reading list will be, it should:

-
    -
  • -

    Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests.

    -
  • -
  • -

    Allow me to tag books instead of placing them into static lists (think clusters or tag clouds).

    -
  • -
  • -

    Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to.

    -
  • -
  • -

    Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it.

    -
  • -
  • -

    Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was.

    -
  • -
  • -

    Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on.

    -
  • -
  • -

    Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from.

    -
  • -
-

Honorable Mentions: None :(

-

Intelligent PDF viewers, eBook readers, audiobook & podcast players # -

-
- -
Functionality I want in my document reader
-
-

Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to:

-
    -
  • -

    Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them.

    -
  • -
  • -

    See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition.

    -
  • -
  • -

    View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know!

    -
  • -
  • -

    Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019.

    -
  • -
  • -

    Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues.

    -
  • -
  • -

    Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points).

    -
  • -
  • -

    Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading.

    -
  • -
-
- -
What I want my audiobook player to look like
-
-

Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could:

-
    -
  • -

    Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street.

    -
  • -
  • -

    View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps?

    -
  • -
-

Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar -

-

A centralized search interface for my digital brain (memex) # -

-

I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything.

-

I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should:

-
    -
  • -

    Accept and parse the following queries:

    -
      -
    • -

      spacex announcement type:video 2016

      -
    • -
    • -

      links from:jon@test.org topic:python

      -
    • -
    • -

      paper on temperature, productivity referenced in book:Uninhabitable Earth

      -
    • -
    • -

      type:pdf habits digital interfaces

      -
    • -
    • -

      reading comprehension type:blog post

      -
    • -
    • -

      printer ink receipt

      -
    • -
    • -

      type:book read:2017 finance

      -
    • -
    • -

      file:py datetime parse

      -
    • -
    -
  • -
  • -

    Respect my privacy: hosted on something I control and never mined for ads.

    -
  • -
  • -

    Support all my devices with two-way sync so I can search and add to it wherever I am.

    -
  • -
  • -

    Be extensible: allow me to easily ingest my own information and extend with desired functionality.

    -
  • -
  • -

    Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of.

    -
  • -
  • -

    Notify me about changes to documents and webpages I’ve visited.

    -
  • -
  • -

    Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it.

    -
  • -
-

Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop -

-

Parting Thoughts # -

-

I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools.

-

It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share.

-

I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback.

-

- HN Discussion -

-

Appendix # -

-

- The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS -

-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html b/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html deleted file mode 100644 index 66ff99044..000000000 --- a/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html +++ /dev/null @@ -1,453 +0,0 @@ - - - - - Digital Tools I Wish Existed :: up & to the right — Jonathan Borichevskiy - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - -
-
-
-

- Digital Tools I Wish Existed -

-

- -
-

- My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later. -

-

- There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed: -

-
    -
  • -

    - The more seamless the acquisition & ingestion, the more engaged I am with the content -

    -
  • -
  • -

    - Insights are just as likely to be found in a 400-page book as in a 40-minute podcast -

    -
  • -
  • -

    - Notes and their subsequent review are essential for long-term retention -

    -
  • -
  • -

    - Recommendations from other humans are as good, if not better, than algorithmic suggestions -

    -
  • -
-

- In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here. -

-

- Queue management for inbound digital content # -

-

- Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance: -

-
    -
  • -

    - Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more. -

    -
  • -
  • -

    - Every book, article, post, or tweet has the potential to lead to more content. -

    -
  • -
  • -

    - Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs. -

    -
  • -
  • -

    - I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee? -

    -
  • -
  • -

    - Learning, work, news, and entertainment all have different priorities in my life (roughly in that order). -

    -
  • -
  • -

    - I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month. -

    -
  • -
  • -

    - I’m not always connected to a stable internet connection. -

    -
  • -
  • -

    - If it’s a long piece of content I want my position saved reliably so I can resume at a later point. -

    -
  • -
  • -

    - I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation. -

    -
  • -
  • -

    - I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place. -

    -
  • -
  • -

    - I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat. -

    -
  • -
-
- -
- Relevant XKCD, as is tradition -
-
-

- Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one. -

-

- Honorable Mentions: Pocket, Instapaper -

-

- A universal book log, recommendation & sharing system # -

-

- I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof. -

-

- Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry. -

-

- Whatever this tool for managing my ever-growing reading list will be, it should: -

-
    -
  • -

    - Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests. -

    -
  • -
  • -

    - Allow me to tag books instead of placing them into static lists (think clusters or tag clouds). -

    -
  • -
  • -

    - Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to. -

    -
  • -
  • -

    - Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it. -

    -
  • -
  • -

    - Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was. -

    -
  • -
  • -

    - Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on. -

    -
  • -
  • -

    - Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from. -

    -
  • -
-

- Honorable Mentions: None :( -

-

- Intelligent PDF viewers, eBook readers, audiobook & podcast players # -

-
- -
- Functionality I want in my document reader -
-
-

- Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to: -

-
    -
  • -

    - Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them. -

    -
  • -
  • -

    - See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition. -

    -
  • -
  • -

    - View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know! -

    -
  • -
  • -

    - Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019. -

    -
  • -
  • -

    - Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues. -

    -
  • -
  • -

    - Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points). -

    -
  • -
  • -

    - Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading. -

    -
  • -
-
- -
- What I want my audiobook player to look like -
-
-

- Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could: -

-
    -
  • -

    - Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street. -

    -
  • -
  • -

    - View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps? -

    -
  • -
-

- Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar -

-

- A centralized search interface for my digital brain (memex) # -

-

- I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything. -

-

- I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should: -

-
    -
  • -

    - Accept and parse the following queries: -

    -
      -
    • -

      - spacex announcement type:video 2016 -

      -
    • -
    • -

      - links from:jon@test.org topic:python -

      -
    • -
    • -

      - paper on temperature, productivity referenced in book:Uninhabitable Earth -

      -
    • -
    • -

      - type:pdf habits digital interfaces -

      -
    • -
    • -

      - reading comprehension type:blog post -

      -
    • -
    • -

      - printer ink receipt -

      -
    • -
    • -

      - type:book read:2017 finance -

      -
    • -
    • -

      - file:py datetime parse -

      -
    • -
    -
  • -
  • -

    - Respect my privacy: hosted on something I control and never mined for ads. -

    -
  • -
  • -

    - Support all my devices with two-way sync so I can search and add to it wherever I am. -

    -
  • -
  • -

    - Be extensible: allow me to easily ingest my own information and extend with desired functionality. -

    -
  • -
  • -

    - Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of. -

    -
  • -
  • -

    - Notify me about changes to documents and webpages I’ve visited. -

    -
  • -
  • -

    - Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it. -

    -
  • -
-

- Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop -

-

- Parting Thoughts # -

-

- I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools. -

-

- It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share. -

-

- I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback. -

-

- HN Discussion -

-

- 2019-12-09: fixes grammar -

-

- Appendix # -

-

- The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS -

-
- -
-
- - - -
- - diff --git a/packages/api/src/services/popular_reads/omnivore_get_started-content.html b/packages/api/src/services/popular_reads/omnivore_get_started-content.html deleted file mode 100644 index d25355959..000000000 --- a/packages/api/src/services/popular_reads/omnivore_get_started-content.html +++ /dev/null @@ -1,331 +0,0 @@ -
-
-
-

Omnivore is a read-it-later app that lets you save and organize everything you read online.

-
-
-

This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities:

-
    -
  • -

    Saving

    -
  • -
  • -

    Reading

    -
  • -
  • -

    Organizing

    -
  • -
  • -

    Integrations

    -
  • -
-

- The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them. -

-

There are five ways to save links to pages or articles that you wish to read later:

-
    -
  • -

    Saving from Your Omnivore Library

    -
  • -
  • -

    Saving from a Browser 

    -
  • -
  • -

    Saving from a Phone or Tablet (iOS or Android)

    -
  • -
  • -

    Newsletter Subscriptions via Email

    -
  • -
  • -

    - Saving PDFs from a Mac
    -

    -
  • -
-

Saving from Your Omnivore Library

-

- 1. In the upper right corner of your Library, tap the Add Link button.
- 2. Enter the URL you wish to save and tap Add Link.
- 3. The link will appear in your Library the next time you refresh it.
-

-

Saving from a Browser

-

1. Download and install the Omnivore extension for your browser:

- -

- 2. Navigate to the page you wish to save and tap the Omnivore button in your browser’s toolbar or Extensions menu.
- 3. Alternatively, you can right-click (command+click on Mac) on any hyperlink and select Save to Omnivore from the menu.
- 4. The link will appear in your Library the next time you refresh it.
-

-

Saving from a Phone or Tablet

-

The best way to save links from your mobile device is via the Omnivore app. You can download the app here:

- -

Once the mobile app is installed:

-
    -
  1. -

    - In your browser, navigate to the page you wish to save and tap the Share button. -

    -
  2. -
  3. -

    - Tap the Omnivore icon in the Share menu. -

    -
  4. -
  5. -

    The link will appear in your Library the next time you refresh it.

    -
  6. -
-

Newsletter Subscriptions via Email

-

- 1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu. -

-

- 2. Tap Create a New Email Address to add a new email address (ex: username-123abc@inbox.omnivore.app) to the list. -

-

3. Click the Copy icon next to the email address.

-

- 4. Navigate to the signup page for the newsletter you wish to subscribe to.
- 5. Paste the Omnivore email address into the signup form. -

-

6. New newsletters will be automatically delivered to your Omnivore inbox.

-

NOTE: If Omnivore receive's an email that does not look like an article, such as a welcome message, or note from the author, it will be forwarded to your Omnivore account email address (the email you registered with).

-

Saving PDFs from a Mac 

-
    -
  1. -

    - Install the Mac App -

    -
  2. -
  3. -

    On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name.

    -
  4. -
  5. -

    - Select Share from the menu and choose Omnivore. -

    -
  6. -
  7. -

    The link will appear in your Library the next time you refresh it.

    -
  8. -
-

Reading

-

Click any link saved in your Library to enter the Reader view. 

-

Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load.

-

While reading, you can:

-
    -
  • -

    Change Formatting

    -
  • -
  • -

    Highlight Text

    -
  • -
  • -

    Add Notes

    -
  • -
  • -

    View All Saved Highlights and Notes

    -
  • -
  • -

    Track Reading Progress

    -
  • -
-

Change Formatting 

-
    -
  1. -

    - Theme: Tap your photo, initial, or avatar  in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme. -

    -
  2. -
  3. -

    - Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing. -

    -
  4. -
-

Highlight Text

-
    -
  1. -

    Select the text you wish to highlight.

    -
  2. -
  3. -

    - Tap the Highlight button. -

    -
  4. -
  5. -

    The text will appear highlighted next time you view the article.

    -
  6. -
-

Add Notes

-
    -
  1. -

    Highlight a section of text where you wish to add a note.

    -
  2. -
  3. -

    - Tap the Note button, type your note, and tap Save. -

    -
  4. -
  5. -

    The Note icon will appear next time you view this article.

    -
  6. -
-

View All Saved Highlights and Notes

-
    -
  1. -

    Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page.

    -
  2. -
  3. -

    To remove a note or highlight, select it from the list and tap the Trash icon.

    -
  4. -
-

Track Reading Progress

-

Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading.

-

Organizing

-

By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions: 

-
    -
  • -

    Archiving

    -
  • -
  • -

    Labels

    -
  • -
  • -

    Search

    -
  • -
  • -

    Filters

    -
  • -
-

Archiving

-
    -
  1. -

    Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu).

    -
  2. -
  3. -

    - Select Archive. -

    -
  4. -
  5. -

    The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below).

    -
  6. -
-

- Labels -

-
    -
  1. -

    - Tap the Menu icon next to any link and select Set Labels. -

    -
  2. -
  3. -

    - Select an existing label from the list or tap Edit Labels to create a new one. -

    -
  4. -
  5. -

    The label will appear next to the link in your Library. Tap it to view all links with the same label.

    -
  6. -
  7. -

    - Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label -

    -
  8. -
  9. -

    Note: Omnivore will automatically assign some labels, such as “Newsletters.”

    -
  10. -
-

Search

-
    -
  1. -

    To search through all your saved links, enter a keyword or phrase in the search bar. 

    -
  2. -
  3. -

    - You can combine keywords with labels and filters to focus your search even further. Learn more about advanced search. -

    -
  4. -
-

Filters

-
    -
  1. -

    - Use the Filters menu to refine your Library view (some filters may be visible by default). -

    -
  2. -
  3. -

    - Select Read Later to view a list of all your non-archived links except Newsletters. -

    -
  4. -
  5. -

    - Select Highlights to view the text selections you have highlighted in all your saved pages.  -

    -
  6. -
  7. -

    - Select Today to view a list of links you saved today. -

    -
  8. -
  9. -

    - Select Newsletters to view links saved via your newsletter subscriptions. -

    -
  10. -
-

Integrations

-

Omnivore allows integrations with knowledge bases and note-taking apps including:

-
    -
  • -

    Logseq

    -
  • -
  • -

    Webhooks

    -
  • -
-

Logseq

-

- With Omnivore's Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide. -

-

Webhooks

-

- Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive. -

-
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_get_started-original.html b/packages/api/src/services/popular_reads/omnivore_get_started-original.html deleted file mode 100644 index b536f4ccd..000000000 --- a/packages/api/src/services/popular_reads/omnivore_get_started-original.html +++ /dev/null @@ -1,1670 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Getting Started with Omnivore - Omnivore - - - - - - - - - - - - - - - - - - - - -
-
- - -
- - - - -
-
-
- -
-
-
-
-
-
-
- -
-
-
- -
-
- Learn the best ways to save links with Omnivore -
-
-
-
- Omnivore -
-
- -
- - - - -
- 2 -
- - - - - -
-
-
-
-
- -
-
- Highlighted <code> in Omnivore -
-
-
-
- Omnivore -
-
- -
- - - - -
- 1 -
- - - - - -
-
-
-
-
- -
-
- Add to your library with your Omnivore email address -
-
-
-
- Omnivore -
-
- -
- - - - -
- 1 -
- - - - - -
-
-
See all - - - - -
-
-
-
-
- -
- -
- -
-
-
- - - - - - - diff --git a/packages/api/src/services/popular_reads/omnivore_ios-content.html b/packages/api/src/services/popular_reads/omnivore_ios-content.html deleted file mode 100644 index 2f79a0603..000000000 --- a/packages/api/src/services/popular_reads/omnivore_ios-content.html +++ /dev/null @@ -1,94 +0,0 @@ -
-
-
-

- With the Omnivore app for iOS, it’s easy to save web pages and articles or archive web content to read later. -

-

- The Omnivore app uses the iOS Share System, which lets you send items from one app (such as Safari) to another (such as Messages or Mail).  -

-
    -
  • -

    Step 1: Log in to the Omnivore app.

    -
  • -
  • -

    Step 2: Add Omnivore to your Share menu favorites.

    -
  • -
  • -

    Step 3: Save links to your Omnivore Library.

    -
  • -
-
-

- -

-
-

You must be logged in before you can save links via the Share menu. If you don’t already have an Omnivore account, you can sign up for free from the login screen.

-

- Note: If you haven’t installed the iOS app, download it here: https://omnivore.app/install/ios -

-

- Step 2: Add Omnivore to your Share menu favorites. -

-

Start by viewing the Share menu from within any supported iOS app (we’ve used Safari for this example).

-
    -
  1. -

    - Tap the Share icon at the bottom of the screen. -

    -
  2. -
  3. -

    - Swipe left to the end of the list of app icons and tap More. -

    -
  4. -
  5. -

    - Tap Edit at the top of the screen. -

    -
  6. -
  7. -

    - Scroll down until you see the Omnivore icon and tap the + icon next to it.  -

    -
  8. -
  9. -

    - Press and hold the three-bar icon and drag Omnivore to one of the top positions under Favorites. Tap Done to close the menu. -

    -
  10. -
  11. -

    - Omnivore will appear as one of the first options the next time you use the Share feature (you may need to restart Safari). -

    -
  12. -
-

- Step 3: Save links to your Omnivore Library -

-

- Start by navigating to the page or article you wish to save. Please note that Omnivore will save the content that appears on your screen (not just a link), so if the page is behind a paywall and you are logged into the paywalled site, you will save the paid content.  -

-
    -
  1. -

    - While viewing the page you’d like to save, tap the Share icon. -

    -
  2. -
  3. -

    - Tap the Omnivore icon in the Share menu. -

    -
  4. -
  5. -

    - Tag the article with one or more labels (optional) and tap Read Now or Read Later -

    -
  6. -
  7. -

    If you choose Read Later, the link will appear in your Library the next time you open the Omnivore app.

    -
  8. -
-
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_ios-original.html b/packages/api/src/services/popular_reads/omnivore_ios-original.html deleted file mode 100644 index 86daf6739..000000000 --- a/packages/api/src/services/popular_reads/omnivore_ios-original.html +++ /dev/null @@ -1,1456 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Saving Links from Your iPhone or iPad - Omnivore - - - - - - - - - - - - - - - - - - - - - -
-
- - -
- - - - -
-
-
- -
-
-
-
-
-
- Comments -
-
-
-
-
- -
-
- -
-
-
-
- - - -
-
-
-
-
-
-
-
- -
-
-
- -
-
- Learn the best ways to save links with Omnivore -
-
-
-
- Omnivore -
-
- -
- - - - -
- 2 -
- - - - - -
-
-
-
-
- -
-
- Highlighted <code> in Omnivore -
-
-
-
- Omnivore -
-
- -
- - - - -
- 1 -
- - - - - -
-
-
-
-
- -
-
- Add to your library with your Omnivore email address -
-
-
-
- Omnivore -
-
- -
- - - - -
- 1 -
- - - - - -
-
-
See all - - - - -
-
-
-
-
- -
- -
- -
-
-
- - - - - - - diff --git a/packages/api/src/services/popular_reads/omnivore_organize-content.html b/packages/api/src/services/popular_reads/omnivore_organize-content.html deleted file mode 100644 index b2ccf3e7c..000000000 --- a/packages/api/src/services/popular_reads/omnivore_organize-content.html +++ /dev/null @@ -1,73 +0,0 @@ -
-
-

Omnivore provides labels (also known as tags) to help you organize your library. Labels can be added to any saved read, and your library can be filtered based on labels.

-

On the web if you have a larger screen you can find the labels tool on the left side of the screen.

-
-
- - - - - -
Adding a label from the left menu
-
-
-

For a smaller screen you will find the labels button at the top of the page.

-
-
- - - - - -
Article Actions at the top of the Omnivore Reader for smaller screens
-
-
-

iOS users can long press on an item in the library or access the labels modal from the menu in the top right of the reader page.

-
-
- - - - - -
Editing labels from the library view on iOS
-
-
-

Label searches on iOS

-

On iOS you can use the label search modal to search for specific labels. This will create an OR search and return all links matching the assigned labels.

-
-
- - - - - -
Using labels to filter your search
-
-
-

Using Advanced Search to filter your library with labels

-

Omnivore's advanced search syntax supports searching for multiple labels using AND and OR clauses. You can also negate a label search to find pages that do not have a certain label.

-

Some examples:

-
    -
  • -

    - label:Newsletter finds all pages that have the label Newsletter -

    -
  • -
  • -

    - label:Cooking,Fitness finds all your pages with either the Cooking or Fitness labels -

    -
  • -
  • -

    - label:Newsletter label:Surfing finds all pages with both the Newsletter and Surfing labels -

    -
  • -
  • -

    label:Coding -label:News finds all pages with the Coding label that do not have the News label

    -
  • -
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_organize-original.html b/packages/api/src/services/popular_reads/omnivore_organize-original.html deleted file mode 100644 index ff3036bc6..000000000 --- a/packages/api/src/services/popular_reads/omnivore_organize-original.html +++ /dev/null @@ -1,1022 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Organize your Omnivore library with labels - Omnivore - - - - - - - - - - - - - - - - - -
-
- - -
- -
-
-
- -
-
-
-
-
-
- -
-
-
- -
-
- Learn the best ways to save links with Omnivore - -
-
-
-
- -
-
- Highlighted <code> in Omnivore - -
-
-
-
- -
-
- Today we are happy to launch our new PDF viewer. It is available in our latest iOS release (1.3.0) and on the web. The new PDF viewer supports… - -
-
See all - - - - -
-
-
-
- -
-
- -
-
- - - - - - - - diff --git a/packages/api/src/services/popular_reads/power_read_it_later-content.html b/packages/api/src/services/popular_reads/power_read_it_later-content.html deleted file mode 100644 index d8366c5f5..000000000 --- a/packages/api/src/services/popular_reads/power_read_it_later-content.html +++ /dev/null @@ -1,239 +0,0 @@ -
-
-
-
-
- -
Image via Nuno Cruz
-
-
-
-

- By Tiago Forte of Forte Labs -

-

At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year.

-

- -

-

This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask.

-

What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place.

-

We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours.

-

The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading.

-

The fact is, the ability to read is becoming a source of competitive advantage in the world.

-

I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time.

-

A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.”

-
-
-

-

Start Building Your Second Brain

-

Subscribe below to learn more about the next cohort of the Building a Second Brain course

-
-
-

Read It. Later.

-

Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar.

-

So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout.

-

On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are:

- -

The app I use, Pocket, adds a button to the Chrome toolbar that looks like this:

-
- -
Chrome toolbar
-
-

- Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. -

-

Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved:

-
-
- -
Mac desktop client
-
-

You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self:

-
-
- -
Web browser “tile” view
-
-

Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces).

-

The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development

-

I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity.

-

But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out.

-

There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with:

-
    -
  1. Increasing consumption of long-form content (which is presumably more substantive)
  2. -
  3. Better filtering
  4. -
-

#1 | Increasing Consumption of Long-Form Content

-

In order to consume good ideas, first you have to consume many ideas.

-

This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation.

-

The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances.

-
-

Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. -

-
-

Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them:

-

1. App performance

-

We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off.

-

If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal.

-

Meanwhile, your snaps and instas refresh at precog-like speeds.

-

Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about.

-

A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off.

-
- -
Effect of ad-blocker on loading times of Boston.com, via NYT -
-
-

Yeah that’s pretty much an eternity in mobile behavior land.

-

2. Matching content with your context

-
-
- -
My Pocket list on iPad
-
-
-

Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse.

-

So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in.

-

Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment.

-

David Allen puts it this way:

-
-

“It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.

-
-

You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓

-

3. Asynchronous reading

-

This is one of the least understood barriers to reading in our fragmented timescape.

-

There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article.

-

The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it.

-

Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished.

-

4. Focus

-

A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it.

-

It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing.

-

Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read.

-
-

Actionable info ≠ Reference info ≠ To Read pile

-
-

Ergo,

-
-

Task manager ≠ Evernote ≠ Pocket

-
-

#2 | Better filtering

-

Now you’ve got the funnel filled. It’s time to narrow it.

-

Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources.

-

The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape.

-

Yeah.

-

In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you.

-

Procrastination is the most powerful force in the universe. It will find a way. -

-

I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time.

-

I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session.

-

So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab.

-

Here’s what a typical Monday morning link-fest looks like, just from email:

-

- -

-

Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing).

-

There’s only one rule: NO READING OR WATCHING! -

-

Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in.

-

I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about.

-

What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me.

-
-

Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide

-
-

Here’s a visual of how this works, from my Pocket analytics:

-

- -

-

You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week.

-

- /sidebar -

-

Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back.

-

I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on.

-

If only this method would scale.

-

- /end_sidebar -

-

Mo’ apps, mo’ problems

-

There are drawbacks, which I’ve glossed over until now. The two main ones:

-

1. Formatting issues

-

Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker.

-

The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed.

-

One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer.

-

2. Dependence

-

Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me.

-

For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved.

-

You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow:

-

- Highlight > Share > Share to Evernote > Save -

-

…has been replaced with this:

-

- Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket -

-

Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date.

-

- /rant_over -

-

Progress Traps and Paradigms

-

The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little.

-

As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know.

-

As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid.

-

What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go?

-

I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.”

-

When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens.

-

Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere.

-

But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on.

-

There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself.

-

Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain.

-

I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms.

-

I can’t put it any better than Donella Meadows, in her seminal piece on complex systems:

-
-

People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. -

-
-
-

It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia.

-
-
-

In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. -

-
-

Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own.

-

The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds.

-

And that is the secret power of Read It Later apps.

-

- P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) -

-
-

Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.

-
-
-

-

Join the Forte Labs Newsletter

-

Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I'll send you my Top 10 All-Time Articles right away as a thank you.

-
-
-
- -
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/power_read_it_later-original.html b/packages/api/src/services/popular_reads/power_read_it_later-original.html deleted file mode 100644 index 69ce2a783..000000000 --- a/packages/api/src/services/popular_reads/power_read_it_later-original.html +++ /dev/null @@ -1,1551 +0,0 @@ - - - - - - - - - - - - The Secret Power of ‘Read It Later’ Apps - Forte Labs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-

- The Secret Power of ‘Read It Later’ Apps -

-
-
- - -
-
-
-

- Estimated reading time: 14 minutes -

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
- Image via Nuno Cruz -
-
-
-
-

- By Tiago Forte of Forte Labs -

-

- At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year. -

-

- -

-

- This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask. -

-

- What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place. -

-

- We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours. -

-

- The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading. -

-

- The fact is, the ability to read is becoming a source of competitive advantage in the world. -

-

- I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time. -

-

- A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.” -

-
-
-
-
-
-

- Start Building Your Second Brain -

-
-
-

- Subscribe below to learn more about the next cohort of the Building a Second Brain course -

-
-
-
-
    -
    -
    - -
    -
    -
    -
    -
    -
    -

    -   -

    -

    - Read It. Later. -

    -

    - Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar. -

    -

    - So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout. -

    -

    - On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are: -

    - -

    - The app I use, Pocket, adds a button to the Chrome toolbar that looks like this: -

    -
    - -
    - Chrome toolbar -
    -
    -

    - Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. -

    -

    - Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved: -

    -
    -
    - -
    - Mac desktop client -
    -
    -

    - You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self: -

    -
    -
    - -
    - Web browser “tile” view -
    -
    -

    - Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces). -

    -

    - The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development -

    -

    - I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity. -

    -

    - But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out. -

    -

    - There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with: -

    -
      -
    1. Increasing consumption of long-form content (which is presumably more substantive) -
    2. -
    3. Better filtering -
    4. -
    -

    - #1 | Increasing Consumption of Long-Form Content -

    -

    - In order to consume good ideas, first you have to consume many ideas. -

    -

    - This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation. -

    -

    - The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances. -

    -
    -

    - Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. -

    -
    -

    - Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them: -

    -

    - 1. App performance -

    -

    - We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off. -

    -

    - If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal. -

    -

    - Meanwhile, your snaps and instas refresh at precog-like speeds. -

    -

    - Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about. -

    -

    - A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off. -

    -
    - -
    - Effect of ad-blocker on loading times of Boston.com, via NYT -
    -
    -

    - Yeah that’s pretty much an eternity in mobile behavior land. -

    -

    - 2. Matching content with your context -

    -
    -
    - -
    - My Pocket list on iPad -
    -
    -
    -

    - Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse. -

    -

    - So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in. -

    -

    - Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment. -

    -

    - David Allen puts it this way: -

    -
    -

    - “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.” -

    -
    -

    - You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓ -

    -

    - 3. Asynchronous reading -

    -

    - This is one of the least understood barriers to reading in our fragmented timescape. -

    -

    - There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article. -

    -

    - The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it. -

    -

    - Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished. -

    -

    - 4. Focus -

    -

    - A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it. -

    -

    - It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing. -

    -

    - Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read. -

    -
    -

    - Actionable info ≠ Reference info ≠ To Read pile -

    -
    -

    - Ergo, -

    -
    -

    - Task manager ≠ Evernote ≠ Pocket -

    -
    -

    - #2 | Better filtering -

    -

    - Now you’ve got the funnel filled. It’s time to narrow it. -

    -

    - Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources. -

    -

    - The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape. -

    -

    - Yeah. -

    -

    - In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you. -

    -

    - Procrastination is the most powerful force in the universe. It will find a way. -

    -

    - I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time. -

    -

    - I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session. -

    -

    - So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab. -

    -

    - Here’s what a typical Monday morning link-fest looks like, just from email: -

    -

    - -

    -

    - Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing). -

    -

    - There’s only one rule: NO READING OR WATCHING! -

    -

    - Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in. -

    -

    - I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about. -

    -

    - What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me. -

    -
    -

    - Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide -

    -
    -

    - Here’s a visual of how this works, from my Pocket analytics: -

    -

    - -

    -

    - You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week. -

    -

    - /sidebar -

    -

    - Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back. -

    -

    - I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on. -

    -

    - If only this method would scale. -

    -

    - /end_sidebar -

    -

    - Mo’ apps, mo’ problems -

    -

    - There are drawbacks, which I’ve glossed over until now. The two main ones: -

    -

    - 1. Formatting issues -

    -

    - Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker. -

    -

    - The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed. -

    -

    - One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer. -

    -

    - 2. Dependence -

    -

    - Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me. -

    -

    - For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved. -

    -

    - You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow: -

    -

    - Highlight > Share > Share to Evernote > Save -

    -

    - …has been replaced with this: -

    -

    - Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket -

    -

    - Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date. -

    -

    - /rant_over -

    -

    - Progress Traps and Paradigms -

    -

    - The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little. -

    -

    - As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know. -

    -

    - As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid. -

    -

    - What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go? -

    -

    - I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.” -

    -

    - When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens. -

    -

    - Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere. -

    -

    - But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on. -

    -

    - There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself. -

    -

    - Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain. -

    -

    - I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms. -

    -

    - I can’t put it any better than Donella Meadows, in her seminal piece on complex systems: -

    -
    -

    - People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. -

    -
    -
    -

    - It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia. -

    -
    -
    -

    - In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. -

    -
    -

    - Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own. -

    -

    - The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds. -

    -

    - And that is the secret power of Read It Later apps. -

    -

    - P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) -

    -
    - Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.
    -
    -
    -
    -
    -
    -
    -
    -

    - Join the Forte Labs Newsletter -

    -
    -
    -

    - Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I'll send you my Top 10 All-Time Articles right away as a thank you. -

    -
    -
    -
    -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - - - -
      -
        -
      • - -
      • -
      • - -
      • -
      • - -
      • -
      • - -
      • -
      • - -
      • -
      • - -
      • -
      -
      -
      - -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/api/src/services/popular_reads/rlove_carnitas-content.html b/packages/api/src/services/popular_reads/rlove_carnitas-content.html deleted file mode 100644 index a02fdb0d6..000000000 --- a/packages/api/src/services/popular_reads/rlove_carnitas-content.html +++ /dev/null @@ -1,34 +0,0 @@ -
      -
      -
      -
      -

      I used to have a bunch of recipes up online. But writing recipes is no fun; it is difficult to capture the beauty of a dish with a bunch of steps. Moreover, using recipes isn’t how I cook. I want to understand the flavors of a dish and then execute it in my own way, in my own hands. So the recipes went away.

      -

      But one of the most popular — and one of my personal favorites — was a recipe for the Mexican pork dish carnitas. It was a fun, relatively easy recipe, not traditional in approach but fairly traditional (and really delicious) in output. Folks keep asking for it. So here it is, in hopes I can eat it at your next house party.

      -

      Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. The traditional recipe is simple: several pounds of pork shoulder, a pound or two of lard, orange peel, and some water (or coca-cola), slow roasted and then “boiled” to a crisp. That is…a bit much. What follows is not an authentic approach.

      -
      -

      -

      -
      Photo by flickr user mccun934 licensed CC BY 2.0 -
      -
      -

      Never ones to eschew taste for health, the French have nonetheless taught us nothing if not that we can make a succulent, flavorful dish without boiling a tough cut of meat in lard. In that vein, my recipe is more of a braiser de porc than a confit de porc — pork shoulder, aromatics, citrus juice, and a little Grand Marnier, slow cooked on the stove. Healthier for the heart, but also — more importantly — tastier to the tongue. To obtain that classic carnitas crisp, we braise the meat uncovered until the liquid evaporates and then move the pot to the oven and caramelize.

      -

      Makes about 6 servings.

      -

      Ingredients:

      -

      3 lbs boneless pork shoulder, preferably Boston butt (the upper shoulder)
      4 tablespoons grape seed oil
      6 cloves garlic, peeled and thinly-sliced
      2 oranges, juiced, plus the zest of 1/2
      2 limes, juiced and zested
      several cups chicken stock
      1 teaspoon ground cayenne pepper
      1 tablespoon ground ancho chile
      1 tablespoon ground chipotle pepper
      2 teaspoons cumin
      1 teaspoon ground cinnamon
      1 teaspoon freshly-ground black pepper
      - herb sachet with 8 sprigs Mexican oregano, 6 sprigs thyme, 2 sticks cinnamon, and 2 bay leaves, wrapped in a cheese cloth and tied shut with cooking twine
      1 cup Grand Marnier
      coarse sea salt, to coat meat, plus more to taste -

      -

      Cut the pork shoulder into 5" chunks. Remove any gratuitously-excessive fat, but leave at least a thin layer. Sprinkle the chunks with sea salt. Let sit at room temperature at least 30 minutes. Pat dry.

      -

      Heat the grape seed oil in a dutch oven over medium-high heat. Sauté the pork shoulder until well browned on each side. If needed, sauté across multiple batches.

      -

      Add the garlic and sauté for one more minute.

      -

      Add the orange zest, lime zest, ground cayenne pepper, ground ancho chile, ground chipotle chile, cumin, ground cinnamon, black pepper, and herb sachet. Mix.

      -

      Add the Grand Marnier, orange juice, and lime juice. Stir, scrapping the bottom of the pot.

      -

      Add chicken stock as needed such that the pork is two-thirds submerged in liquid.

      -

      Stirring, raise heat to high and bring to a boil. Once boiling, lower heat until the liquid is at a simmer and braise, uncovered, stirring occasionally, until the pork is cooked and tender but not disintegrating and the liquid is reduced by at least two-thirds, about three hours. If the liquid gets perilously-low while cooking, add a little chicken stock.

      -

      Remove pork from pot. Once cool enough to handle, use a fork to shred the pork into bite-sized, but fairly large, chunks, removing any fatty pieces as desired.

      -

      Preheat oven to 450°F.

      -

      Return pork chunks to pot. Place uncovered pot in oven. Continue cooking until the liquid has evaporated and the pork is crispy and starting to caramelize, about 20 minutes.

      -

      Taste and adjust salt. Serve with warm corn tortillas, guacamole, pico de gallo, diced white onion, chopped cilantro, lime wedges, margaritas, and college football.

      -
      -
      -
      -
      \ No newline at end of file diff --git a/packages/api/src/services/popular_reads/rlove_carnitas-original.html b/packages/api/src/services/popular_reads/rlove_carnitas-original.html deleted file mode 100644 index e4db70b00..000000000 --- a/packages/api/src/services/popular_reads/rlove_carnitas-original.html +++ /dev/null @@ -1,1230 +0,0 @@ - - - - - Slow-Braised Carnitas Recipe. Robert Love’s recipe for carnitas, in… | by Robert Love | Medium - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      -
      -
      - -
      - -
      -
      -
      - -
      -
      - -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -

      - Slow-Braised Carnitas Recipe -

      -
      - - - -
      -
      - -
      -
      - Photo by flickr user mccun934 licensed CC BY 2.0 -
      -
      - - - - - - - - - - - - - - - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -

      - More from Robert Love -

      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -

      - Google, Android, Linux kernel, author, Boston -

      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -

      - Love podcasts or audiobooks? Learn on the go with our new app. -

      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - - - -
      - -

      - Get the Medium app -

      -
      -
      - A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store -
      A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - - -
      -
      -
      -
      -
      - -
      -
      -
      - Robert Love -
      -
      -
      -

      - Robert Love -

      -
      -
      -

      - Google, Android, Linux kernel, author, Boston -

      -
      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -

      - More from Medium -

      -
      - -
      -

      - This Produce Season: Winter -

      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - - -
      -
      -

      - Eli’s Level 2 Board & Train: Small Dog with Big Dog Syndrome -

      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -

      - in -

      -
      - -
      -
      -

      - Masala Chai Recipe -

      -
      -
      -
      -
      -
      - A cup of masala chai (Indian spiced tea) in a green plate -
      -
      -
      -
      -
      - - - -
      -

      - in -

      -
      - -
      -

      - Vegemite Spaghetti Recipe — An Easy One Pot Dinner -

      -
      -
      -
      -
      - Vegemite pasta spaghetti recipe -
      -
      -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/db/migrations/0140.do.popular_read.sql b/packages/db/migrations/0140.do.popular_read.sql new file mode 100755 index 000000000..1c769c652 --- /dev/null +++ b/packages/db/migrations/0140.do.popular_read.sql @@ -0,0 +1,9931 @@ +-- Type: DO +-- Name: popular_read +-- Description: Create omnivore.popular_read table + +BEGIN; + +CREATE TABLE omnivore.popular_read ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + key text NOT NULL, + original_url text NOT NULL, + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + thumbnail text, + published_at timestamptz, + site_name text NOT NULL, + slug text NOT NULL, + readable_content text NOT NULL, + original_content text NOT NULL, + word_count integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + UNIQUE (key) +); + +CREATE TRIGGER update_popular_read_modtime BEFORE UPDATE ON omnivore.popular_read FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +GRANT SELECT ON omnivore.popular_read TO omnivore_user; + +INSERT INTO omnivore.popular_read (key, original_url, title, author, description, thumbnail, published_at, site_name, slug, readable_content, original_content, word_count) +VALUES + ('omnivore_get_started', 'https://blog.omnivore.app/p/getting-started-with-omnivore', 'Getting Started with Omnivore', 'The Omnivore Team', 'Get the most out of Omnivore by learning how to use it.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sxQnqya1QNApB7ZAGPj9K20AU6sw0UAnjmAIy2ub8hUU/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F658efff4-341a-4720-8cf6-9b2bdbedfaa7_800x668.gif', '2021-10-13', 'Omnivore Blog', 'getting-started-with-omnivore', '
      +
      +
      +

      Omnivore is a read-it-later app that lets you save and organize everything you read online.

      +
      +
      +

      This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities:

      +
        +
      • +

        Saving

        +
      • +
      • +

        Reading

        +
      • +
      • +

        Organizing

        +
      • +
      • +

        Integrations

        +
      • +
      +

      + The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them. +

      +

      There are five ways to save links to pages or articles that you wish to read later:

      +
        +
      • +

        Saving from Your Omnivore Library

        +
      • +
      • +

        Saving from a Browser 

        +
      • +
      • +

        Saving from a Phone or Tablet (iOS or Android)

        +
      • +
      • +

        Newsletter Subscriptions via Email

        +
      • +
      • +

        + Saving PDFs from a Mac
        +

        +
      • +
      +

      Saving from Your Omnivore Library

      +

      + 1. In the upper right corner of your Library, tap the Add Link button.
      + 2. Enter the URL you wish to save and tap Add Link.
      + 3. The link will appear in your Library the next time you refresh it.
      +

      +

      Saving from a Browser

      +

      1. Download and install the Omnivore extension for your browser:

      + +

      + 2. Navigate to the page you wish to save and tap the Omnivore button in your browser’s toolbar or Extensions menu.
      + 3. Alternatively, you can right-click (command+click on Mac) on any hyperlink and select Save to Omnivore from the menu.
      + 4. The link will appear in your Library the next time you refresh it.
      +

      +

      Saving from a Phone or Tablet

      +

      The best way to save links from your mobile device is via the Omnivore app. You can download the app here:

      + +

      Once the mobile app is installed:

      +
        +
      1. +

        + In your browser, navigate to the page you wish to save and tap the Share button. +

        +
      2. +
      3. +

        + Tap the Omnivore icon in the Share menu. +

        +
      4. +
      5. +

        The link will appear in your Library the next time you refresh it.

        +
      6. +
      +

      Newsletter Subscriptions via Email

      +

      + 1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu. +

      +

      + 2. Tap Create a New Email Address to add a new email address (ex: username-123abc@inbox.omnivore.app) to the list. +

      +

      3. Click the Copy icon next to the email address.

      +

      + 4. Navigate to the signup page for the newsletter you wish to subscribe to.
      + 5. Paste the Omnivore email address into the signup form. +

      +

      6. New newsletters will be automatically delivered to your Omnivore inbox.

      +

      NOTE: If Omnivore receive's an email that does not look like an article, such as a welcome message, or note from the author, it will be forwarded to your Omnivore account email address (the email you registered with).

      +

      Saving PDFs from a Mac 

      +
        +
      1. +

        + Install the Mac App +

        +
      2. +
      3. +

        On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name.

        +
      4. +
      5. +

        + Select Share from the menu and choose Omnivore. +

        +
      6. +
      7. +

        The link will appear in your Library the next time you refresh it.

        +
      8. +
      +

      Reading

      +

      Click any link saved in your Library to enter the Reader view. 

      +

      Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load.

      +

      While reading, you can:

      +
        +
      • +

        Change Formatting

        +
      • +
      • +

        Highlight Text

        +
      • +
      • +

        Add Notes

        +
      • +
      • +

        View All Saved Highlights and Notes

        +
      • +
      • +

        Track Reading Progress

        +
      • +
      +

      Change Formatting 

      +
        +
      1. +

        + Theme: Tap your photo, initial, or avatar  in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme. +

        +
      2. +
      3. +

        + Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing. +

        +
      4. +
      +

      Highlight Text

      +
        +
      1. +

        Select the text you wish to highlight.

        +
      2. +
      3. +

        + Tap the Highlight button. +

        +
      4. +
      5. +

        The text will appear highlighted next time you view the article.

        +
      6. +
      +

      Add Notes

      +
        +
      1. +

        Highlight a section of text where you wish to add a note.

        +
      2. +
      3. +

        + Tap the Note button, type your note, and tap Save. +

        +
      4. +
      5. +

        The Note icon will appear next time you view this article.

        +
      6. +
      +

      View All Saved Highlights and Notes

      +
        +
      1. +

        Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page.

        +
      2. +
      3. +

        To remove a note or highlight, select it from the list and tap the Trash icon.

        +
      4. +
      +

      Track Reading Progress

      +

      Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading.

      +

      Organizing

      +

      By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions: 

      +
        +
      • +

        Archiving

        +
      • +
      • +

        Labels

        +
      • +
      • +

        Search

        +
      • +
      • +

        Filters

        +
      • +
      +

      Archiving

      +
        +
      1. +

        Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu).

        +
      2. +
      3. +

        + Select Archive. +

        +
      4. +
      5. +

        The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below).

        +
      6. +
      +

      + Labels +

      +
        +
      1. +

        + Tap the Menu icon next to any link and select Set Labels. +

        +
      2. +
      3. +

        + Select an existing label from the list or tap Edit Labels to create a new one. +

        +
      4. +
      5. +

        The label will appear next to the link in your Library. Tap it to view all links with the same label.

        +
      6. +
      7. +

        + Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label +

        +
      8. +
      9. +

        Note: Omnivore will automatically assign some labels, such as “Newsletters.”

        +
      10. +
      +

      Search

      +
        +
      1. +

        To search through all your saved links, enter a keyword or phrase in the search bar. 

        +
      2. +
      3. +

        + You can combine keywords with labels and filters to focus your search even further. Learn more about advanced search. +

        +
      4. +
      +

      Filters

      +
        +
      1. +

        + Use the Filters menu to refine your Library view (some filters may be visible by default). +

        +
      2. +
      3. +

        + Select Read Later to view a list of all your non-archived links except Newsletters. +

        +
      4. +
      5. +

        + Select Highlights to view the text selections you have highlighted in all your saved pages.  +

        +
      6. +
      7. +

        + Select Today to view a list of links you saved today. +

        +
      8. +
      9. +

        + Select Newsletters to view links saved via your newsletter subscriptions. +

        +
      10. +
      +

      Integrations

      +

      Omnivore allows integrations with knowledge bases and note-taking apps including:

      +
        +
      • +

        Logseq

        +
      • +
      • +

        Webhooks

        +
      • +
      +

      Logseq

      +

      + With Omnivore''s Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide. +

      +

      Webhooks

      +

      + Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive. +

      +
      +
      +
      +', ' + + + + + + + + + + + + + + + + + + + + + + + + Getting Started with Omnivore - Omnivore + + + + + + + + + + + + + + + + + + + + +
      +
      + + +
      + + + + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 2 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Add to your library with your Omnivore email address +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      See all + + + + +
      +
      +
      +
      +
      + +
      + +
      + +
      +
      +
      + + + + + + + +', 1155), + ('omnivore_ios', 'https://blog.omnivore.app/p/saving-links-from-your-iphone-or', 'Saving Links from Your iPhone or iPad', 'Omnivore', 'Learn how to save articles on iOS.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sWDfv7sARTIdAlx6Rw_6t-QwL3T9aniEJRa1-jVaglNg/https://substackcdn.com/image/youtube/w_728,c_limit/k6RkIqepAig', '2021-10-19', 'Omnivore Blog', 'saving-links-from-your-i-phone-or-i-pad', '
      +
      +
      +

      + With the Omnivore app for iOS, it’s easy to save web pages and articles or archive web content to read later. +

      +

      + The Omnivore app uses the iOS Share System, which lets you send items from one app (such as Safari) to another (such as Messages or Mail).  +

      +
        +
      • +

        Step 1: Log in to the Omnivore app.

        +
      • +
      • +

        Step 2: Add Omnivore to your Share menu favorites.

        +
      • +
      • +

        Step 3: Save links to your Omnivore Library.

        +
      • +
      +
      +

      + +

      +
      +

      You must be logged in before you can save links via the Share menu. If you don’t already have an Omnivore account, you can sign up for free from the login screen.

      +

      + Note: If you haven’t installed the iOS app, download it here: https://omnivore.app/install/ios +

      +

      + Step 2: Add Omnivore to your Share menu favorites. +

      +

      Start by viewing the Share menu from within any supported iOS app (we’ve used Safari for this example).

      +
        +
      1. +

        + Tap the Share icon at the bottom of the screen. +

        +
      2. +
      3. +

        + Swipe left to the end of the list of app icons and tap More. +

        +
      4. +
      5. +

        + Tap Edit at the top of the screen. +

        +
      6. +
      7. +

        + Scroll down until you see the Omnivore icon and tap the + icon next to it.  +

        +
      8. +
      9. +

        + Press and hold the three-bar icon and drag Omnivore to one of the top positions under Favorites. Tap Done to close the menu. +

        +
      10. +
      11. +

        + Omnivore will appear as one of the first options the next time you use the Share feature (you may need to restart Safari). +

        +
      12. +
      +

      + Step 3: Save links to your Omnivore Library +

      +

      + Start by navigating to the page or article you wish to save. Please note that Omnivore will save the content that appears on your screen (not just a link), so if the page is behind a paywall and you are logged into the paywalled site, you will save the paid content.  +

      +
        +
      1. +

        + While viewing the page you’d like to save, tap the Share icon. +

        +
      2. +
      3. +

        + Tap the Omnivore icon in the Share menu. +

        +
      4. +
      5. +

        + Tag the article with one or more labels (optional) and tap Read Now or Read Later +

        +
      6. +
      7. +

        If you choose Read Later, the link will appear in your Library the next time you open the Omnivore app.

        +
      8. +
      +
      +
      +
      ', ' + + + + + + + + + + + + + + + + + + + + + + + + Saving Links from Your iPhone or iPad - Omnivore + + + + + + + + + + + + + + + + + + + + + +
      +
      + + +
      + + + + +
      +
      +
      + +
      +
      +
      +
      +
      +
      + Comments +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      + + + +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 2 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Add to your library with your Omnivore email address +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      See all + + + + +
      +
      +
      +
      +
      + +
      + +
      + +
      +
      +
      + + + + + + + +', 371), + ('omnivore_organize', 'https://blog.omnivore.app/p/organize-your-omnivore-library-with', 'Organize your Omnivore library with labels', 'The Omnivore Team', 'Use labels to organize your Omnivore library.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sTgJ5Q0XIg_EHdmPWcxtXFmkjn8T6hkJt7S9ziClagYo/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fdaf07af7-5cdb-4ecc-aace-1a46de3e9c58_1827x1090.png', '2022-04-18', 'Omnivore Blog', 'organize-your-omnivore-library-with-labels', '
      +
      +

      Omnivore provides labels (also known as tags) to help you organize your library. Labels can be added to any saved read, and your library can be filtered based on labels.

      +

      On the web if you have a larger screen you can find the labels tool on the left side of the screen.

      +
      +
      + + + + + +
      Adding a label from the left menu
      +
      +
      +

      For a smaller screen you will find the labels button at the top of the page.

      +
      +
      + + + + + +
      Article Actions at the top of the Omnivore Reader for smaller screens
      +
      +
      +

      iOS users can long press on an item in the library or access the labels modal from the menu in the top right of the reader page.

      +
      +
      + + + + + +
      Editing labels from the library view on iOS
      +
      +
      +

      Label searches on iOS

      +

      On iOS you can use the label search modal to search for specific labels. This will create an OR search and return all links matching the assigned labels.

      +
      +
      + + + + + +
      Using labels to filter your search
      +
      +
      +

      Using Advanced Search to filter your library with labels

      +

      Omnivore''s advanced search syntax supports searching for multiple labels using AND and OR clauses. You can also negate a label search to find pages that do not have a certain label.

      +

      Some examples:

      +
        +
      • +

        + label:Newsletter finds all pages that have the label Newsletter +

        +
      • +
      • +

        + label:Cooking,Fitness finds all your pages with either the Cooking or Fitness labels +

        +
      • +
      • +

        + label:Newsletter label:Surfing finds all pages with both the Newsletter and Surfing labels +

        +
      • +
      • +

        label:Coding -label:News finds all pages with the Coding label that do not have the News label

        +
      • +
      +
      +
      +', ' + + + + + + + + + + + + + + + + + + + + + + + + Organize your Omnivore library with labels - Omnivore + + + + + + + + + + + + + + + + + +
      +
      + + +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore + +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore + +
      +
      +
      +
      + +
      +
      + Today we are happy to launch our new PDF viewer. It is available in our latest iOS release (1.3.0) and on the web. The new PDF viewer supports… + +
      +
      See all + + + + +
      +
      +
      +
      + +
      +
      + +
      +
      + + + + + + + + +', 259), + ('rlove_carnitas', 'https://medium.com/@rlove/carnitas-ff0ef1044ae9', 'Slow-Braised Carnitas Recipe', 'Robert Love', 'Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. This is not an authentic recipe.', 'https://proxy-prod.omnivore-image-cache.app/88x88,sIcDXt3Ar0baKG1e1Yi1e2VUZFL85xPlOeEfAxF-s-Nw/https://miro.medium.com/max/1200/1*Wl-dMBJpSgPUxUOnPQthyg.jpeg', '2017-02-24', '@rlove', 'slow-braised-carnitas-recipe', '
      +
      +
      +
      +

      I used to have a bunch of recipes up online. But writing recipes is no fun; it is difficult to capture the beauty of a dish with a bunch of steps. Moreover, using recipes isn’t how I cook. I want to understand the flavors of a dish and then execute it in my own way, in my own hands. So the recipes went away.

      +

      But one of the most popular — and one of my personal favorites — was a recipe for the Mexican pork dish carnitas. It was a fun, relatively easy recipe, not traditional in approach but fairly traditional (and really delicious) in output. Folks keep asking for it. So here it is, in hopes I can eat it at your next house party.

      +

      Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. The traditional recipe is simple: several pounds of pork shoulder, a pound or two of lard, orange peel, and some water (or coca-cola), slow roasted and then “boiled” to a crisp. That is…a bit much. What follows is not an authentic approach.

      +
      +

      +

      +
      Photo by flickr user mccun934 licensed CC BY 2.0 +
      +
      +

      Never ones to eschew taste for health, the French have nonetheless taught us nothing if not that we can make a succulent, flavorful dish without boiling a tough cut of meat in lard. In that vein, my recipe is more of a braiser de porc than a confit de porc — pork shoulder, aromatics, citrus juice, and a little Grand Marnier, slow cooked on the stove. Healthier for the heart, but also — more importantly — tastier to the tongue. To obtain that classic carnitas crisp, we braise the meat uncovered until the liquid evaporates and then move the pot to the oven and caramelize.

      +

      Makes about 6 servings.

      +

      Ingredients:

      +

      3 lbs boneless pork shoulder, preferably Boston butt (the upper shoulder)
      4 tablespoons grape seed oil
      6 cloves garlic, peeled and thinly-sliced
      2 oranges, juiced, plus the zest of 1/2
      2 limes, juiced and zested
      several cups chicken stock
      1 teaspoon ground cayenne pepper
      1 tablespoon ground ancho chile
      1 tablespoon ground chipotle pepper
      2 teaspoons cumin
      1 teaspoon ground cinnamon
      1 teaspoon freshly-ground black pepper
      + herb sachet with 8 sprigs Mexican oregano, 6 sprigs thyme, 2 sticks cinnamon, and 2 bay leaves, wrapped in a cheese cloth and tied shut with cooking twine
      1 cup Grand Marnier
      coarse sea salt, to coat meat, plus more to taste +

      +

      Cut the pork shoulder into 5" chunks. Remove any gratuitously-excessive fat, but leave at least a thin layer. Sprinkle the chunks with sea salt. Let sit at room temperature at least 30 minutes. Pat dry.

      +

      Heat the grape seed oil in a dutch oven over medium-high heat. Sauté the pork shoulder until well browned on each side. If needed, sauté across multiple batches.

      +

      Add the garlic and sauté for one more minute.

      +

      Add the orange zest, lime zest, ground cayenne pepper, ground ancho chile, ground chipotle chile, cumin, ground cinnamon, black pepper, and herb sachet. Mix.

      +

      Add the Grand Marnier, orange juice, and lime juice. Stir, scrapping the bottom of the pot.

      +

      Add chicken stock as needed such that the pork is two-thirds submerged in liquid.

      +

      Stirring, raise heat to high and bring to a boil. Once boiling, lower heat until the liquid is at a simmer and braise, uncovered, stirring occasionally, until the pork is cooked and tender but not disintegrating and the liquid is reduced by at least two-thirds, about three hours. If the liquid gets perilously-low while cooking, add a little chicken stock.

      +

      Remove pork from pot. Once cool enough to handle, use a fork to shred the pork into bite-sized, but fairly large, chunks, removing any fatty pieces as desired.

      +

      Preheat oven to 450°F.

      +

      Return pork chunks to pot. Place uncovered pot in oven. Continue cooking until the liquid has evaporated and the pork is crispy and starting to caramelize, about 20 minutes.

      +

      Taste and adjust salt. Serve with warm corn tortillas, guacamole, pico de gallo, diced white onion, chopped cilantro, lime wedges, margaritas, and college football.

      +
      +
      +
      +
      ', ' + + + + Slow-Braised Carnitas Recipe. Robert Love’s recipe for carnitas, in… | by Robert Love | Medium + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      +
      + +
      + +
      +
      +
      + +
      +
      + +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +

      + Slow-Braised Carnitas Recipe +

      +
      + + + +
      +
      + +
      +
      + Photo by flickr user mccun934 licensed CC BY 2.0 +
      +
      + + + + + + + + + + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + More from Robert Love +

      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +

      + Google, Android, Linux kernel, author, Boston +

      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + Love podcasts or audiobooks? Learn on the go with our new app. +

      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + + + +
      + +

      + Get the Medium app +

      +
      +
      + A button that says ''Download on the App Store'', and if clicked it will lead you to the iOS App store +
      A button that says ''Get it on, Google Play'', and if clicked it will lead you to the Google Play store +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + + +
      +
      +
      +
      +
      + +
      +
      +
      + Robert Love +
      +
      +
      +

      + Robert Love +

      +
      +
      +

      + Google, Android, Linux kernel, author, Boston +

      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +

      + More from Medium +

      +
      + +
      +

      + This Produce Season: Winter +

      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + + +
      +
      +

      + Eli’s Level 2 Board & Train: Small Dog with Big Dog Syndrome +

      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +

      + in +

      +
      + +
      +
      +

      + Masala Chai Recipe +

      +
      +
      +
      +
      +
      + A cup of masala chai (Indian spiced tea) in a green plate +
      +
      +
      +
      +
      + + + +
      +

      + in +

      +
      + +
      +

      + Vegemite Spaghetti Recipe — An Easy One Pot Dinner +

      +
      +
      +
      +
      + Vegemite pasta spaghetti recipe +
      +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +', 705), + ('power_read_it_later', 'https://fortelabs.co/blog/the-secret-power-of-read-it-later-apps', 'The Secret Power of ‘Read It Later’ Apps', 'Tiago Forte', 'At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket', 'https://proxy-prod.omnivore-image-cache.app/320x320,sGN5R34M5z068QMXDZD32CQD6mCbxc47hWXm__JVUePE/https://fortelabs.com/wp-content/uploads/2015/11/1rPXwIczUJRCE54v8FfAHGw.jpeg', '2022-01-24', 'Forte Labs', 'the-secret-power-of-read-it-later-apps', '
      +
      +
      +
      +
      + +
      Image via Nuno Cruz
      +
      +
      +
      +

      + By Tiago Forte of Forte Labs +

      +

      At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year.

      +

      + +

      +

      This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask.

      +

      What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place.

      +

      We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours.

      +

      The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading.

      +

      The fact is, the ability to read is becoming a source of competitive advantage in the world.

      +

      I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time.

      +

      A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.”

      +
      +
      +

      +

      Start Building Your Second Brain

      +

      Subscribe below to learn more about the next cohort of the Building a Second Brain course

      +
      +
      +

      Read It. Later.

      +

      Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar.

      +

      So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout.

      +

      On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are:

      + +

      The app I use, Pocket, adds a button to the Chrome toolbar that looks like this:

      +
      + +
      Chrome toolbar
      +
      +

      + Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. +

      +

      Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved:

      +
      +
      + +
      Mac desktop client
      +
      +

      You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self:

      +
      +
      + +
      Web browser “tile” view
      +
      +

      Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces).

      +

      The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development

      +

      I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity.

      +

      But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out.

      +

      There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with:

      +
        +
      1. Increasing consumption of long-form content (which is presumably more substantive)
      2. +
      3. Better filtering
      4. +
      +

      #1 | Increasing Consumption of Long-Form Content

      +

      In order to consume good ideas, first you have to consume many ideas.

      +

      This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation.

      +

      The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances.

      +
      +

      Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. +

      +
      +

      Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them:

      +

      1. App performance

      +

      We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off.

      +

      If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal.

      +

      Meanwhile, your snaps and instas refresh at precog-like speeds.

      +

      Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about.

      +

      A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off.

      +
      + +
      Effect of ad-blocker on loading times of Boston.com, via NYT +
      +
      +

      Yeah that’s pretty much an eternity in mobile behavior land.

      +

      2. Matching content with your context

      +
      +
      + +
      My Pocket list on iPad
      +
      +
      +

      Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse.

      +

      So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in.

      +

      Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment.

      +

      David Allen puts it this way:

      +
      +

      “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.

      +
      +

      You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓

      +

      3. Asynchronous reading

      +

      This is one of the least understood barriers to reading in our fragmented timescape.

      +

      There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article.

      +

      The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it.

      +

      Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished.

      +

      4. Focus

      +

      A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it.

      +

      It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing.

      +

      Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read.

      +
      +

      Actionable info ≠ Reference info ≠ To Read pile

      +
      +

      Ergo,

      +
      +

      Task manager ≠ Evernote ≠ Pocket

      +
      +

      #2 | Better filtering

      +

      Now you’ve got the funnel filled. It’s time to narrow it.

      +

      Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources.

      +

      The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape.

      +

      Yeah.

      +

      In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you.

      +

      Procrastination is the most powerful force in the universe. It will find a way. +

      +

      I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time.

      +

      I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session.

      +

      So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab.

      +

      Here’s what a typical Monday morning link-fest looks like, just from email:

      +

      + +

      +

      Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing).

      +

      There’s only one rule: NO READING OR WATCHING! +

      +

      Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in.

      +

      I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about.

      +

      What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me.

      +
      +

      Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide

      +
      +

      Here’s a visual of how this works, from my Pocket analytics:

      +

      + +

      +

      You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week.

      +

      + /sidebar +

      +

      Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back.

      +

      I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on.

      +

      If only this method would scale.

      +

      + /end_sidebar +

      +

      Mo’ apps, mo’ problems

      +

      There are drawbacks, which I’ve glossed over until now. The two main ones:

      +

      1. Formatting issues

      +

      Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker.

      +

      The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed.

      +

      One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer.

      +

      2. Dependence

      +

      Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me.

      +

      For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved.

      +

      You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow:

      +

      + Highlight > Share > Share to Evernote > Save +

      +

      …has been replaced with this:

      +

      + Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket +

      +

      Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date.

      +

      + /rant_over +

      +

      Progress Traps and Paradigms

      +

      The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little.

      +

      As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know.

      +

      As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid.

      +

      What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go?

      +

      I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.”

      +

      When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens.

      +

      Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere.

      +

      But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on.

      +

      There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself.

      +

      Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain.

      +

      I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms.

      +

      I can’t put it any better than Donella Meadows, in her seminal piece on complex systems:

      +
      +

      People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. +

      +
      +
      +

      It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia.

      +
      +
      +

      In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. +

      +
      +

      Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own.

      +

      The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds.

      +

      And that is the secret power of Read It Later apps.

      +

      + P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) +

      +
      +

      Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.

      +
      +
      +

      +

      Join the Forte Labs Newsletter

      +

      Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I''ll send you my Top 10 All-Time Articles right away as a thank you.

      +
      +
      +
      + +
      +
      +', ' + + + + + + + + + + + The Secret Power of ‘Read It Later’ Apps - Forte Labs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + The Secret Power of ‘Read It Later’ Apps +

      +
      +
      + + +
      +
      +
      +

      + Estimated reading time: 14 minutes +

      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      + Image via Nuno Cruz +
      +
      +
      +
      +

      + By Tiago Forte of Forte Labs +

      +

      + At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year. +

      +

      + +

      +

      + This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask. +

      +

      + What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place. +

      +

      + We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours. +

      +

      + The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading. +

      +

      + The fact is, the ability to read is becoming a source of competitive advantage in the world. +

      +

      + I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time. +

      +

      + A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.” +

      +
      +
      +
      +
      +
      +

      + Start Building Your Second Brain +

      +
      +
      +

      + Subscribe below to learn more about the next cohort of the Building a Second Brain course +

      +
      +
      +
      +
        +
        +
        + +
        +
        +
        +
        +
        +
        +

        +   +

        +

        + Read It. Later. +

        +

        + Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar. +

        +

        + So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout. +

        +

        + On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are: +

        + +

        + The app I use, Pocket, adds a button to the Chrome toolbar that looks like this: +

        +
        + +
        + Chrome toolbar +
        +
        +

        + Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. +

        +

        + Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved: +

        +
        +
        + +
        + Mac desktop client +
        +
        +

        + You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self: +

        +
        +
        + +
        + Web browser “tile” view +
        +
        +

        + Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces). +

        +

        + The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development +

        +

        + I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity. +

        +

        + But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out. +

        +

        + There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with: +

        +
          +
        1. Increasing consumption of long-form content (which is presumably more substantive) +
        2. +
        3. Better filtering +
        4. +
        +

        + #1 | Increasing Consumption of Long-Form Content +

        +

        + In order to consume good ideas, first you have to consume many ideas. +

        +

        + This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation. +

        +

        + The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances. +

        +
        +

        + Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. +

        +
        +

        + Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them: +

        +

        + 1. App performance +

        +

        + We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off. +

        +

        + If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal. +

        +

        + Meanwhile, your snaps and instas refresh at precog-like speeds. +

        +

        + Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about. +

        +

        + A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off. +

        +
        + +
        + Effect of ad-blocker on loading times of Boston.com, via NYT +
        +
        +

        + Yeah that’s pretty much an eternity in mobile behavior land. +

        +

        + 2. Matching content with your context +

        +
        +
        + +
        + My Pocket list on iPad +
        +
        +
        +

        + Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse. +

        +

        + So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in. +

        +

        + Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment. +

        +

        + David Allen puts it this way: +

        +
        +

        + “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.” +

        +
        +

        + You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓ +

        +

        + 3. Asynchronous reading +

        +

        + This is one of the least understood barriers to reading in our fragmented timescape. +

        +

        + There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article. +

        +

        + The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it. +

        +

        + Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished. +

        +

        + 4. Focus +

        +

        + A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it. +

        +

        + It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing. +

        +

        + Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read. +

        +
        +

        + Actionable info ≠ Reference info ≠ To Read pile +

        +
        +

        + Ergo, +

        +
        +

        + Task manager ≠ Evernote ≠ Pocket +

        +
        +

        + #2 | Better filtering +

        +

        + Now you’ve got the funnel filled. It’s time to narrow it. +

        +

        + Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources. +

        +

        + The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape. +

        +

        + Yeah. +

        +

        + In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you. +

        +

        + Procrastination is the most powerful force in the universe. It will find a way. +

        +

        + I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time. +

        +

        + I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session. +

        +

        + So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab. +

        +

        + Here’s what a typical Monday morning link-fest looks like, just from email: +

        +

        + +

        +

        + Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing). +

        +

        + There’s only one rule: NO READING OR WATCHING! +

        +

        + Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in. +

        +

        + I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about. +

        +

        + What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me. +

        +
        +

        + Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide +

        +
        +

        + Here’s a visual of how this works, from my Pocket analytics: +

        +

        + +

        +

        + You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week. +

        +

        + /sidebar +

        +

        + Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back. +

        +

        + I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on. +

        +

        + If only this method would scale. +

        +

        + /end_sidebar +

        +

        + Mo’ apps, mo’ problems +

        +

        + There are drawbacks, which I’ve glossed over until now. The two main ones: +

        +

        + 1. Formatting issues +

        +

        + Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker. +

        +

        + The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed. +

        +

        + One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer. +

        +

        + 2. Dependence +

        +

        + Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me. +

        +

        + For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved. +

        +

        + You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow: +

        +

        + Highlight > Share > Share to Evernote > Save +

        +

        + …has been replaced with this: +

        +

        + Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket +

        +

        + Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date. +

        +

        + /rant_over +

        +

        + Progress Traps and Paradigms +

        +

        + The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little. +

        +

        + As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know. +

        +

        + As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid. +

        +

        + What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go? +

        +

        + I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.” +

        +

        + When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens. +

        +

        + Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere. +

        +

        + But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on. +

        +

        + There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself. +

        +

        + Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain. +

        +

        + I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms. +

        +

        + I can’t put it any better than Donella Meadows, in her seminal piece on complex systems: +

        +
        +

        + People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. +

        +
        +
        +

        + It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia. +

        +
        +
        +

        + In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. +

        +
        +

        + Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own. +

        +

        + The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds. +

        +

        + And that is the secret power of Read It Later apps. +

        +

        + P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) +

        +
        + Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.
        +
        +
        +
        +
        +
        +
        +
        +

        + Join the Forte Labs Newsletter +

        +
        +
        +

        + Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I''ll send you my Top 10 All-Time Articles right away as a thank you. +

        +
        +
        +
        +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + +
          +
            +
          • + +
          • +
          • + +
          • +
          • + +
          • +
          • + +
          • +
          • + +
          • +
          • + +
          • +
          +
          +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +', 3726), + ('elad_meetings', 'http://blog.eladgil.com/2018/07/meeting-etiquette.html', 'Better Meetings', 'Elad Gil', 'How to make meetings more productive.', null, '2018-07-02', 'Elad Blog', 'better-meetings', '
          +
          +

          As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
          +
          1. Determine who is necessary in the meeting.
          + Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
          +
          2. Send out an agenda in advance.
          + What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
          +
          Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
          +
          3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
          + If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
          +
          If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
          +
          4. Kick off the meeting with objectives.
          + Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
          +
          5. Assign a note taker.
          + Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
          +
          6. Send out meeting notes.
          + Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
          +
          Meeting notes would optimally include:
          + a. Subject/topic of meeting
          + b. Date
          + c. Attendees
          + d. Actions/decisions
          + e. Agenda
          + f. Detailed notes
          +
          7. Clean up the meeting calendar ongoing.

          + Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
          +
          NOTES

          + [1] There may be legal reasons to not take notes in specific meetings. Consult with the company''s GC or external counsel on training for meeting note etiquette. +

          +

          MY BOOK +

          +

          You can pre-order the High Growth Handbook here.
          +
          +

          +
          +
          +', ' + + + + + + + + + + + + + + + + + Elad Blog: Better Meetings + + + + + + + + + + + + + + + + + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +

          + Monday, July 2, 2018 +

          +
          +
          +
          + + +

          + Better Meetings +

          +
          +
          +
          +
          +
          + As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
          +
          1. Determine who is necessary in the meeting.
          + Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
          +
          2. Send out an agenda in advance.
          + What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
          +
          Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
          +
          3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
          + If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
          +
          If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
          +
          4. Kick off the meeting with objectives.
          + Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
          +
          5. Assign a note taker.
          + Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
          +
          6. Send out meeting notes.
          + Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
          +
          Meeting notes would optimally include:
          + a. Subject/topic of meeting
          + b. Date
          + c. Attendees
          + d. Actions/decisions
          + e. Agenda
          + f. Detailed notes
          +
          7. Clean up the meeting calendar ongoing.

          + Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
          +
          NOTES

          + [1] There may be legal reasons to not take notes in specific meetings. Consult with the company''s GC or external counsel on training for meeting note etiquette.
          +
          +
          +
          +
          + MY BOOK +
          You can pre-order the High Growth Handbook here.
          +

          +
          +
          +
          +
          + RELATED POSTS +
          + + + + + + +
          +
          +
          + +
          +
          + +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + + + + +', 545), + ('jonbo_digital_tools', 'https://jon.bo/posts/digital-tools/', 'Digital Tools I Wish Existed', 'Jonathan Borichevskiy', 'My digital life in a nutshell: I discover relevant content I don’t have time to consume...', null, '2019-11-28', 'JON.BO', 'digital-tools-i-wish-existed', '
          +
          +

          My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later.

          +

          There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed:

          +
            +
          • +

            The more seamless the acquisition & ingestion, the more engaged I am with the content

            +
          • +
          • +

            Insights are just as likely to be found in a 400-page book as in a 40-minute podcast

            +
          • +
          • +

            Notes and their subsequent review are essential for long-term retention

            +
          • +
          • +

            Recommendations from other humans are as good, if not better, than algorithmic suggestions

            +
          • +
          +

          In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here.

          +

          Queue management for inbound digital content # +

          +

          Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance:

          +
            +
          • +

            Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more.

            +
          • +
          • +

            Every book, article, post, or tweet has the potential to lead to more content.

            +
          • +
          • +

            Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs.

            +
          • +
          • +

            I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee?

            +
          • +
          • +

            Learning, work, news, and entertainment all have different priorities in my life (roughly in that order).

            +
          • +
          • +

            I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month.

            +
          • +
          • +

            I’m not always connected to a stable internet connection.

            +
          • +
          • +

            If it’s a long piece of content I want my position saved reliably so I can resume at a later point.

            +
          • +
          • +

            I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation.

            +
          • +
          • +

            I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place.

            +
          • +
          • +

            I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat.

            +
          • +
          +
          + +
          Relevant XKCD, as is tradition
          +
          +

          Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one.

          +

          Honorable Mentions: Pocket, Instapaper +

          +

          A universal book log, recommendation & sharing system # +

          +

          I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof.

          +

          Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry.

          +

          Whatever this tool for managing my ever-growing reading list will be, it should:

          +
            +
          • +

            Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests.

            +
          • +
          • +

            Allow me to tag books instead of placing them into static lists (think clusters or tag clouds).

            +
          • +
          • +

            Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to.

            +
          • +
          • +

            Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it.

            +
          • +
          • +

            Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was.

            +
          • +
          • +

            Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on.

            +
          • +
          • +

            Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from.

            +
          • +
          +

          Honorable Mentions: None :(

          +

          Intelligent PDF viewers, eBook readers, audiobook & podcast players # +

          +
          + +
          Functionality I want in my document reader
          +
          +

          Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to:

          +
            +
          • +

            Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them.

            +
          • +
          • +

            See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition.

            +
          • +
          • +

            View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know!

            +
          • +
          • +

            Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019.

            +
          • +
          • +

            Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues.

            +
          • +
          • +

            Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points).

            +
          • +
          • +

            Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading.

            +
          • +
          +
          + +
          What I want my audiobook player to look like
          +
          +

          Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could:

          +
            +
          • +

            Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street.

            +
          • +
          • +

            View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps?

            +
          • +
          +

          Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar +

          +

          A centralized search interface for my digital brain (memex) # +

          +

          I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything.

          +

          I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should:

          +
            +
          • +

            Accept and parse the following queries:

            +
              +
            • +

              spacex announcement type:video 2016

              +
            • +
            • +

              links from:jon@test.org topic:python

              +
            • +
            • +

              paper on temperature, productivity referenced in book:Uninhabitable Earth

              +
            • +
            • +

              type:pdf habits digital interfaces

              +
            • +
            • +

              reading comprehension type:blog post

              +
            • +
            • +

              printer ink receipt

              +
            • +
            • +

              type:book read:2017 finance

              +
            • +
            • +

              file:py datetime parse

              +
            • +
            +
          • +
          • +

            Respect my privacy: hosted on something I control and never mined for ads.

            +
          • +
          • +

            Support all my devices with two-way sync so I can search and add to it wherever I am.

            +
          • +
          • +

            Be extensible: allow me to easily ingest my own information and extend with desired functionality.

            +
          • +
          • +

            Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of.

            +
          • +
          • +

            Notify me about changes to documents and webpages I’ve visited.

            +
          • +
          • +

            Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it.

            +
          • +
          +

          Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop +

          +

          Parting Thoughts # +

          +

          I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools.

          +

          It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share.

          +

          I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback.

          +

          + HN Discussion +

          +

          Appendix # +

          +

          + The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS +

          +
          +
          ', ' + + + + Digital Tools I Wish Existed :: up & to the right — Jonathan Borichevskiy + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + + +
          +
          +
          +

          + Digital Tools I Wish Existed +

          +

          + +
          +

          + My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later. +

          +

          + There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed: +

          +
            +
          • +

            + The more seamless the acquisition & ingestion, the more engaged I am with the content +

            +
          • +
          • +

            + Insights are just as likely to be found in a 400-page book as in a 40-minute podcast +

            +
          • +
          • +

            + Notes and their subsequent review are essential for long-term retention +

            +
          • +
          • +

            + Recommendations from other humans are as good, if not better, than algorithmic suggestions +

            +
          • +
          +

          + In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here. +

          +

          + Queue management for inbound digital content # +

          +

          + Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance: +

          +
            +
          • +

            + Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more. +

            +
          • +
          • +

            + Every book, article, post, or tweet has the potential to lead to more content. +

            +
          • +
          • +

            + Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs. +

            +
          • +
          • +

            + I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee? +

            +
          • +
          • +

            + Learning, work, news, and entertainment all have different priorities in my life (roughly in that order). +

            +
          • +
          • +

            + I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month. +

            +
          • +
          • +

            + I’m not always connected to a stable internet connection. +

            +
          • +
          • +

            + If it’s a long piece of content I want my position saved reliably so I can resume at a later point. +

            +
          • +
          • +

            + I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation. +

            +
          • +
          • +

            + I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place. +

            +
          • +
          • +

            + I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat. +

            +
          • +
          +
          + +
          + Relevant XKCD, as is tradition +
          +
          +

          + Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one. +

          +

          + Honorable Mentions: Pocket, Instapaper +

          +

          + A universal book log, recommendation & sharing system # +

          +

          + I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof. +

          +

          + Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry. +

          +

          + Whatever this tool for managing my ever-growing reading list will be, it should: +

          +
            +
          • +

            + Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests. +

            +
          • +
          • +

            + Allow me to tag books instead of placing them into static lists (think clusters or tag clouds). +

            +
          • +
          • +

            + Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to. +

            +
          • +
          • +

            + Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it. +

            +
          • +
          • +

            + Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was. +

            +
          • +
          • +

            + Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on. +

            +
          • +
          • +

            + Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from. +

            +
          • +
          +

          + Honorable Mentions: None :( +

          +

          + Intelligent PDF viewers, eBook readers, audiobook & podcast players # +

          +
          + +
          + Functionality I want in my document reader +
          +
          +

          + Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to: +

          +
            +
          • +

            + Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them. +

            +
          • +
          • +

            + See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition. +

            +
          • +
          • +

            + View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know! +

            +
          • +
          • +

            + Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019. +

            +
          • +
          • +

            + Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues. +

            +
          • +
          • +

            + Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points). +

            +
          • +
          • +

            + Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading. +

            +
          • +
          +
          + +
          + What I want my audiobook player to look like +
          +
          +

          + Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could: +

          +
            +
          • +

            + Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street. +

            +
          • +
          • +

            + View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps? +

            +
          • +
          +

          + Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar +

          +

          + A centralized search interface for my digital brain (memex) # +

          +

          + I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything. +

          +

          + I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should: +

          +
            +
          • +

            + Accept and parse the following queries: +

            +
              +
            • +

              + spacex announcement type:video 2016 +

              +
            • +
            • +

              + links from:jon@test.org topic:python +

              +
            • +
            • +

              + paper on temperature, productivity referenced in book:Uninhabitable Earth +

              +
            • +
            • +

              + type:pdf habits digital interfaces +

              +
            • +
            • +

              + reading comprehension type:blog post +

              +
            • +
            • +

              + printer ink receipt +

              +
            • +
            • +

              + type:book read:2017 finance +

              +
            • +
            • +

              + file:py datetime parse +

              +
            • +
            +
          • +
          • +

            + Respect my privacy: hosted on something I control and never mined for ads. +

            +
          • +
          • +

            + Support all my devices with two-way sync so I can search and add to it wherever I am. +

            +
          • +
          • +

            + Be extensible: allow me to easily ingest my own information and extend with desired functionality. +

            +
          • +
          • +

            + Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of. +

            +
          • +
          • +

            + Notify me about changes to documents and webpages I’ve visited. +

            +
          • +
          • +

            + Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it. +

            +
          • +
          +

          + Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop +

          +

          + Parting Thoughts # +

          +

          + I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools. +

          +

          + It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share. +

          +

          + I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback. +

          +

          + HN Discussion +

          +

          + 2019-12-09: fixes grammar +

          +

          + Appendix # +

          +

          + The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS +

          +
          + +
          +
          + + + +
          + + +', 2285); + +COMMIT; diff --git a/packages/db/migrations/0140.undo.popular_read.sql b/packages/db/migrations/0140.undo.popular_read.sql new file mode 100755 index 000000000..c45addf8a --- /dev/null +++ b/packages/db/migrations/0140.undo.popular_read.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: popular_read +-- Description: Create omnivore.popular_read table + +BEGIN; + +DROP TABLE IF EXISTS omnivore.popular_read; + +COMMIT; From 3fa3526a166d8e62e9c601d978c0cfda2407fd06 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 15:16:02 +0800 Subject: [PATCH 78/78] fix wrong order of parameters in update scription api integration --- packages/rss-handler/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 28a87bd3d..e3b8d896d 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -241,9 +241,9 @@ const getLink = (links: RssFeedItemLink[]) => { } const processSubscription = async ( - feedUrl: string, - userId: string, subscriptionId: string, + userId: string, + feedUrl: string, fetchResult: { content: string; checksum: string }, lastFetchedAt: number, scheduledAt: number,