From 67c485786364e3db5eb4cb54bd5c509569de1216 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:54:49 +0000 Subject: [PATCH 001/115] Bump graphql from 16.6.0 to 16.8.1 in /apple/gql-server Bumps [graphql](https://github.com/graphql/graphql-js) from 16.6.0 to 16.8.1. - [Release notes](https://github.com/graphql/graphql-js/releases) - [Commits](https://github.com/graphql/graphql-js/compare/v16.6.0...v16.8.1) --- updated-dependencies: - dependency-name: graphql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- apple/gql-server/package.json | 2 +- apple/gql-server/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apple/gql-server/package.json b/apple/gql-server/package.json index 3337f3b4b..9088164b4 100644 --- a/apple/gql-server/package.json +++ b/apple/gql-server/package.json @@ -9,6 +9,6 @@ "dependencies": { "express": "^4.18.1", "express-graphql": "^0.12.0", - "graphql": "^16.4.0" + "graphql": "^16.8.1" } } \ No newline at end of file diff --git a/apple/gql-server/yarn.lock b/apple/gql-server/yarn.lock index ae72deb0f..587c08d1d 100644 --- a/apple/gql-server/yarn.lock +++ b/apple/gql-server/yarn.lock @@ -194,10 +194,10 @@ get-intrinsic@^1.0.2: has "^1.0.3" has-symbols "^1.0.3" -graphql@^16.4.0: - version "16.6.0" - resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.6.0.tgz#c2dcffa4649db149f6282af726c8c83f1c7c5fdb" - integrity sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw== +graphql@^16.8.1: + version "16.8.1" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.8.1.tgz#1930a965bef1170603702acdb68aedd3f3cf6f07" + integrity sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw== has-symbols@^1.0.3: version "1.0.3" From 8f24f17da87201bf616b7b5ab6b36703ed502fe6 Mon Sep 17 00:00:00 2001 From: Remy Chantenay Date: Thu, 12 Oct 2023 08:36:33 +0200 Subject: [PATCH 002/115] Add label name length validation upon creation --- .../ui/components/LabelsSelectionSheet.kt | 29 +++++++++++---- .../omnivore/ui/components/LabelsViewModel.kt | 37 ++++++++++--------- .../app/src/main/res/values/strings.xml | 1 + 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt index 0461149a7..7022e28f8 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt @@ -3,6 +3,7 @@ package app.omnivore.omnivore.ui.components import LabelChip +import android.widget.Toast import androidx.compose.foundation.* import androidx.compose.foundation.interaction.FocusInteraction import androidx.compose.foundation.interaction.MutableInteractionSource @@ -341,19 +342,33 @@ fun LabelsSelectionSheetContent( ) if (!isLibraryMode && filterTextValue.text.isNotEmpty() && currentLabel == null) { + val context = LocalContext.current Row( horizontalArrangement = Arrangement.Start, verticalAlignment = Alignment.CenterVertically, modifier = Modifier .fillMaxWidth() .clickable { - val label = findOrCreateLabel( - labelsViewModel = labelsViewModel, - labels = labels, - name = filterTextValue - ) - state.addChip(LabelChipView(label)) - filterTextValue = TextFieldValue() + when(labelsViewModel.validateLabelName(filterTextValue.text)) { + LabelsViewModel.Error.LabelNameTooLong -> { + Toast.makeText( + context, + context.getString(R.string.label_selection_sheet_label_too_long_error_msg, + labelsViewModel.labelNameMaxLength), + Toast.LENGTH_SHORT + ).show() + } + null -> { + val label = findOrCreateLabel( + labelsViewModel = labelsViewModel, + labels = labels, + name = filterTextValue + ) + + state.addChip(LabelChipView(label)) + filterTextValue = TextFieldValue() + } + } } .padding(horizontal = 10.dp) .padding(top = 10.dp, bottom = 5.dp) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt index f75959f70..7b26a05df 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt @@ -1,34 +1,16 @@ package app.omnivore.omnivore.ui.components - -import android.content.ClipData -import android.content.ClipboardManager -import android.content.Context -import android.content.Intent -import android.net.Uri import android.util.Log -import android.widget.Toast -import androidx.compose.ui.platform.LocalContext -import androidx.core.content.ContextCompat.startActivity import androidx.lifecycle.* -import app.omnivore.omnivore.DatastoreKeys import app.omnivore.omnivore.DatastoreRepository import app.omnivore.omnivore.dataService.* import app.omnivore.omnivore.graphql.generated.type.CreateLabelInput -import app.omnivore.omnivore.graphql.generated.type.SetLabelsInput import app.omnivore.omnivore.models.ServerSyncStatus import app.omnivore.omnivore.networking.* -import app.omnivore.omnivore.persistence.entities.SavedItem -import app.omnivore.omnivore.persistence.entities.SavedItemAndSavedItemLabelCrossRef import app.omnivore.omnivore.persistence.entities.SavedItemLabel -import app.omnivore.omnivore.ui.components.LabelSwatchHelper -import app.omnivore.omnivore.ui.library.SavedItemAction import com.apollographql.apollo3.api.Optional.Companion.presentIfNotNull -import com.google.gson.Gson import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.* -import kotlinx.coroutines.flow.distinctUntilChanged -import java.time.Instant import java.time.LocalDate import java.time.ZoneOffset import java.time.format.DateTimeFormatter @@ -42,6 +24,25 @@ class LabelsViewModel @Inject constructor( private val dataService: DataService, private val networker: Networker ): ViewModel() { + val labelNameMaxLength = 64 + + enum class Error { + LabelNameTooLong + } + + /** + * Checks whether or not the provided label name is valid. + * @param labelName The name of the label. + * @return null if valid, [Error] otherwise. + */ + fun validateLabelName(labelName: String): Error? { + val trimmedName = labelName.trim() + if (trimmedName.count() > labelNameMaxLength) { + return Error.LabelNameTooLong + } + + return null + } fun createNewSavedItemLabelWithTemp(labelName: String, hexColorValue: String): SavedItemLabel { val tempId = UUID.randomUUID().toString() diff --git a/android/Omnivore/app/src/main/res/values/strings.xml b/android/Omnivore/app/src/main/res/values/strings.xml index ad7f2a3ea..2867be8e3 100644 --- a/android/Omnivore/app/src/main/res/values/strings.xml +++ b/android/Omnivore/app/src/main/res/values/strings.xml @@ -103,6 +103,7 @@ Search Save Create a new label named \"%1$s\" + The name provided is too long (must be less or equal than %1$d characters) Labels From 4ae6c476f819b27ec85e168b44e85bb988c3960e Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Sat, 14 Oct 2023 13:18:40 +0800 Subject: [PATCH 003/115] shows api error message --- packages/rule-handler/src/filter.ts | 13 ++++++++++--- packages/rule-handler/src/rule.ts | 1 - 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/rule-handler/src/filter.ts b/packages/rule-handler/src/filter.ts index a3c43e902..2eb365c05 100644 --- a/packages/rule-handler/src/filter.ts +++ b/packages/rule-handler/src/filter.ts @@ -6,6 +6,9 @@ interface SearchResponse { edges: Edge[] } } + errors?: { + message: string + }[] } interface Edge { @@ -28,7 +31,6 @@ interface Label { } export const search = async ( - userId: string, apiEndpoint: string, auth: string, query: string @@ -75,6 +77,12 @@ export const search = async ( } ) + if (response.data.errors) { + console.error(response.data.errors) + + return [] + } + const edges = response.data.data.search.edges if (edges.length === 0) { return [] @@ -89,14 +97,13 @@ export const search = async ( } export const filterPage = async ( - userId: string, apiEndpoint: string, auth: string, filter: string, pageId: string ): Promise => { filter += ` includes:${pageId}` - const pages = await search(userId, apiEndpoint, auth, filter) + const pages = await search(apiEndpoint, auth, filter) return pages.length > 0 ? pages[0] : null } diff --git a/packages/rule-handler/src/rule.ts b/packages/rule-handler/src/rule.ts index b7345fb2c..ef80aa973 100644 --- a/packages/rule-handler/src/rule.ts +++ b/packages/rule-handler/src/rule.ts @@ -93,7 +93,6 @@ export const triggerActions = async ( } const filteredPage = await filterPage( - userId, apiEndpoint, authToken, rule.filter, From 668485ba15b75b04a76b34942e707b5bcd31d881 Mon Sep 17 00:00:00 2001 From: Remy Chantenay Date: Sat, 14 Oct 2023 12:57:35 +0200 Subject: [PATCH 004/115] Trim labelname upon creation --- .../ui/components/LabelsSelectionSheet.kt | 15 ++++++++------- .../omnivore/ui/components/LabelsViewModel.kt | 3 +-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt index 7022e28f8..7dc98a9f7 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt @@ -220,12 +220,12 @@ class LabelChipView(label: SavedItemLabel) : Chip(label.name) { val label = label } -fun findOrCreateLabel(labelsViewModel: LabelsViewModel, labels: List, name: TextFieldValue): SavedItemLabel { - val found = labels.find { it.name == name.text } +fun findOrCreateLabel(labelsViewModel: LabelsViewModel, labels: List, name: String): SavedItemLabel { + val found = labels.find { it.name == name } if (found != null) { return found } - return labelsViewModel.createNewSavedItemLabelWithTemp(name.text, LabelSwatchHelper.random()) + return labelsViewModel.createNewSavedItemLabelWithTemp(name, LabelSwatchHelper.random()) } @Composable @@ -309,7 +309,7 @@ fun LabelsSelectionSheetContent( LabelChipView(it) } ?: null } else { - LabelChipView(findOrCreateLabel(labelsViewModel = labelsViewModel, labels = labels, name = it)) + LabelChipView(findOrCreateLabel(labelsViewModel = labelsViewModel, labels = labels, name = it.text)) } }, chipLeadingIcon = { chip -> CircleIcon(colorHex = chip.label.color) }, @@ -349,7 +349,8 @@ fun LabelsSelectionSheetContent( modifier = Modifier .fillMaxWidth() .clickable { - when(labelsViewModel.validateLabelName(filterTextValue.text)) { + val labelName = filterTextValue.text.trim() + when(labelsViewModel.validateLabelName(labelName)) { LabelsViewModel.Error.LabelNameTooLong -> { Toast.makeText( context, @@ -362,7 +363,7 @@ fun LabelsSelectionSheetContent( val label = findOrCreateLabel( labelsViewModel = labelsViewModel, labels = labels, - name = filterTextValue + name = labelName ) state.addChip(LabelChipView(label)) @@ -379,7 +380,7 @@ fun LabelsSelectionSheetContent( contentDescription = null, modifier = Modifier.padding(end = 8.dp) ) - Text(text = stringResource(R.string.label_selection_sheet_text_create, filterTextValue.text)) + Text(text = stringResource(R.string.label_selection_sheet_text_create, filterTextValue.text.trim())) } } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt index 7b26a05df..ddcdaf07b 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt @@ -36,8 +36,7 @@ class LabelsViewModel @Inject constructor( * @return null if valid, [Error] otherwise. */ fun validateLabelName(labelName: String): Error? { - val trimmedName = labelName.trim() - if (trimmedName.count() > labelNameMaxLength) { + if (labelName.count() > labelNameMaxLength) { return Error.LabelNameTooLong } From f1a71ae86c9ec98291f6a138bfb188e30a8fb876 Mon Sep 17 00:00:00 2001 From: Remy Chantenay Date: Mon, 16 Oct 2023 09:05:56 +0200 Subject: [PATCH 005/115] Android: Show URL in OpenLinkView --- .../omnivore/ui/reader/OpenLinkView.kt | 59 +++++++++++++++++++ .../ui/reader/WebReaderLoadingContainer.kt | 36 ----------- 2 files changed, 59 insertions(+), 36 deletions(-) create mode 100644 android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt new file mode 100644 index 000000000..4ac6f8545 --- /dev/null +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt @@ -0,0 +1,59 @@ +package app.omnivore.omnivore.ui.reader + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Button +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import app.omnivore.omnivore.R + +@Composable +fun OpenLinkView(webReaderViewModel: WebReaderViewModel) { + val context = LocalContext.current + + Column(modifier = Modifier + .padding(top = 25.dp) + .padding(horizontal = 50.dp), verticalArrangement = Arrangement.spacedBy(20.dp)) { + Row { + Text(webReaderViewModel.currentLink.toString(), + fontWeight = FontWeight.Light, + color = Color.DarkGray, + fontSize = 12.sp, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + Row(modifier = Modifier.padding(top = 25.dp)) { + Button(onClick = { webReaderViewModel.openCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_open_in_browser)) + } + } + Row { + Button(onClick = { webReaderViewModel.saveCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_save_to_omnivore)) + + } + } + Row { + Button(onClick = {webReaderViewModel.copyCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_copy_link)) + } + } + Row { + Button(onClick = {webReaderViewModel.resetBottomSheet() }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_cancel)) + } + } + } +} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt index e680a42a7..87a0d63ed 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt @@ -42,7 +42,6 @@ import com.google.accompanist.systemuicontroller.rememberSystemUiController import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.launch import kotlin.math.roundToInt -import androidx.compose.material3.Button import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import app.omnivore.omnivore.ui.components.LabelsViewModel @@ -450,38 +449,3 @@ fun BottomSheetUI(title: String?, content: @Composable () -> Unit) { } } } - -@OptIn(ExperimentalMaterial3Api::class, ExperimentalMaterialApi::class) -@Composable -fun OpenLinkView(webReaderViewModel: WebReaderViewModel) { - val context = LocalContext.current - - Column(modifier = Modifier - .padding(top = 50.dp) - .padding(horizontal = 50.dp), verticalArrangement = Arrangement.spacedBy(20.dp)) { - Row { - Button(onClick = { webReaderViewModel.openCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_open_in_browser)) - - } - } - Row() { - Button(onClick = { webReaderViewModel.saveCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_save_to_omnivore)) - - } - } - Row() { - Button(onClick = {webReaderViewModel.copyCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_copy_link)) - - } - } - Row { - Button(onClick = {webReaderViewModel.resetBottomSheet() }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_cancel)) - - } - } - } -} From e54c1c81a163ae503b73f69db0955724533884f6 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 16 Oct 2023 15:24:53 +0800 Subject: [PATCH 006/115] soft delete user by calling delete account api --- packages/api/src/entity/user.ts | 1 + packages/api/src/resolvers/user/index.ts | 14 +++-- packages/api/src/services/user.ts | 2 +- packages/api/test/resolvers/user.test.ts | 56 ++++++++++++++++++- .../0135.do.alter_user_status_type.sql | 9 +++ .../0135.undo.alter_user_status_type.sql | 9 +++ 6 files changed, 85 insertions(+), 6 deletions(-) create mode 100755 packages/db/migrations/0135.do.alter_user_status_type.sql create mode 100755 packages/db/migrations/0135.undo.alter_user_status_type.sql diff --git a/packages/api/src/entity/user.ts b/packages/api/src/entity/user.ts index a926b5a89..83526f946 100644 --- a/packages/api/src/entity/user.ts +++ b/packages/api/src/entity/user.ts @@ -22,6 +22,7 @@ export enum RegistrationType { export enum StatusType { Active = 'ACTIVE', Pending = 'PENDING', + Deleted = 'DELETED', } @Entity() diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index 8b80e1340..bf99a5c04 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -1,5 +1,9 @@ import * as jwt from 'jsonwebtoken' -import { RegistrationType, User as UserEntity } from '../../entity/user' +import { + RegistrationType, + StatusType, + User as UserEntity, +} from '../../entity/user' import { env } from '../../env' import { DeleteAccountError, @@ -38,6 +42,7 @@ import { import { userRepository } from '../../repository/user' import { createUser } from '../../services/create_user' import { sendVerificationEmail } from '../../services/send_emails' +import { updateUser } from '../../services/user' import { authorized, userDataToUser } from '../../utils/helpers' import { validateUsername } from '../../utils/usernamePolicy' import { WithDataSourcesContext } from '../types' @@ -313,9 +318,10 @@ export const deleteAccountResolver = authorized< DeleteAccountSuccess, DeleteAccountError, MutationDeleteAccountArgs ->(async (_, { userID }, { authTrx, log }) => { - const result = await authTrx(async (t) => { - return t.withRepository(userRepository).delete(userID) +>(async (_, { userID }, { log }) => { + // soft delete user + const result = await updateUser(userID, { + status: StatusType.Deleted, }) if (!result.affected) { log.error('Error deleting user account') diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index aed61b8f9..39babc270 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -13,7 +13,7 @@ export const deleteUser = async (userId: string) => { } export const updateUser = async (userId: string, update: Partial) => { - await authTrx( + return authTrx( async (t) => t.getRepository(User).update(userId, update), undefined, userId diff --git a/packages/api/test/resolvers/user.test.ts b/packages/api/test/resolvers/user.test.ts index 404f4a09d..82e0ceefc 100644 --- a/packages/api/test/resolvers/user.test.ts +++ b/packages/api/test/resolvers/user.test.ts @@ -9,7 +9,7 @@ import { findProfile } from '../../src/services/profile' import { deleteUser, findUser } from '../../src/services/user' import { hashPassword } from '../../src/utils/auth' import { createTestUser } from '../db' -import { graphqlRequest, request } from '../util' +import { generateFakeUuid, graphqlRequest, request } from '../util' describe('User API', () => { const correctPassword = 'fakePassword' @@ -238,4 +238,58 @@ describe('User API', () => { return graphqlRequest(query, invalidAuthToken).expect(500) }) }) + + describe('Delete account', () => { + const query = (userId: string) => ` + mutation { + deleteAccount( + userID: "${userId}" + ) { + ... on DeleteAccountSuccess { + userID + } + ... on DeleteAccountError { + errorCodes + } + } + } + ` + + let userId: string + let authToken: string + + before(async () => { + const user = await createTestUser('to_delete_user') + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + userId = user.id + authToken = res.body.authToken + }) + + after(async () => { + await deleteUser(userId) + }) + + context('when user id is valid', () => { + it('deletes user and responds with 200', async () => { + const response = await graphqlRequest(query(userId), authToken).expect( + 200 + ) + expect(response.body.data.deleteAccount.userID).to.eql(userId) + }) + }) + + context('when user not found', () => { + it('responds with error code UserNotFound', async () => { + const response = await graphqlRequest( + query(generateFakeUuid()), + authToken + ).expect(200) + expect(response.body.data.deleteAccount.errorCodes).to.eql([ + 'USER_NOT_FOUND', + ]) + }) + }) + }) }) diff --git a/packages/db/migrations/0135.do.alter_user_status_type.sql b/packages/db/migrations/0135.do.alter_user_status_type.sql new file mode 100755 index 000000000..da4f39eef --- /dev/null +++ b/packages/db/migrations/0135.do.alter_user_status_type.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: alter_user_status_type +-- Description: Add DELETED to the user_status_type enum + +BEGIN; + +ALTER TYPE user_status_type ADD VALUE 'DELETED'; + +COMMIT; diff --git a/packages/db/migrations/0135.undo.alter_user_status_type.sql b/packages/db/migrations/0135.undo.alter_user_status_type.sql new file mode 100755 index 000000000..a39f1f7ba --- /dev/null +++ b/packages/db/migrations/0135.undo.alter_user_status_type.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: alter_user_status_type +-- Description: Add DELETED to the user_status_type enum + +BEGIN; + +ALTER TYPE user_status_type DROP VALUE IF EXISTS 'DELETED'; + +COMMIT; From 1b2d93e11842c87103c2ec00f093952f5f7f93a7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 16 Oct 2023 17:35:29 +0800 Subject: [PATCH 007/115] remove duplicate rss subscriptions by user and url; create a unique constraint --- packages/api/src/repository/user.ts | 4 +-- .../importers/uploadImportFileResolver.ts | 4 +-- .../src/resolvers/recommendations/index.ts | 8 ++---- packages/api/src/resolvers/save/index.ts | 26 ++++++------------ .../send_install_instructions/index.ts | 6 ++--- .../api/src/resolvers/subscriptions/index.ts | 3 ++- packages/api/src/resolvers/user/index.ts | 27 +++++++++---------- packages/api/src/routers/auth/auth_router.ts | 4 +-- .../api/src/routers/auth/mobile/sign_in.ts | 2 +- packages/api/src/routers/user_router.ts | 2 +- packages/api/src/services/newsletters.ts | 5 +--- packages/api/src/services/save_url.ts | 4 +-- packages/api/src/services/user.ts | 4 +-- packages/api/src/utils/parser.ts | 2 ++ .../api/test/resolvers/subscriptions.test.ts | 2 +- ...0136.do.add_unique_to_subscription_url.sql | 19 +++++++++++++ ...36.undo.add_unique_to_subscription_url.sql | 9 +++++++ 17 files changed, 70 insertions(+), 61 deletions(-) create mode 100755 packages/db/migrations/0136.do.add_unique_to_subscription_url.sql create mode 100755 packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql diff --git a/packages/api/src/repository/user.ts b/packages/api/src/repository/user.ts index 408543c95..a8f327b0d 100644 --- a/packages/api/src/repository/user.ts +++ b/packages/api/src/repository/user.ts @@ -1,6 +1,6 @@ import { In } from 'typeorm' import { appDataSource } from '../data_source' -import { User } from './../entity/user' +import { StatusType, User } from './../entity/user' const TOP_USERS = [ 'jacksonh', @@ -16,7 +16,7 @@ export const MAX_RECORDS_LIMIT = 1000 export const userRepository = appDataSource.getRepository(User).extend({ findById(id: string) { - return this.findOneBy({ id }) + return this.findOneBy({ id, status: StatusType.Active }) }, findByEmail(email: string) { diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts index 4f327ce54..efa88f9b9 100644 --- a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -33,14 +33,14 @@ export const uploadImportFileResolver = authorized< UploadImportFileSuccess, UploadImportFileError, MutationUploadImportFileArgs ->(async (_, { type, contentType }, { claims: { uid }, log }) => { +>(async (_, { type, contentType }, { uid }) => { if (!VALID_CONTENT_TYPES.includes(contentType)) { return { errorCodes: [UploadImportFileErrorCode.BadRequest], } } - const user = await userRepository.findOneBy({ id: uid }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [UploadImportFileErrorCode.Unauthorized], diff --git a/packages/api/src/resolvers/recommendations/index.ts b/packages/api/src/resolvers/recommendations/index.ts index d3062dc5d..31f7da898 100644 --- a/packages/api/src/resolvers/recommendations/index.ts +++ b/packages/api/src/resolvers/recommendations/index.ts @@ -48,9 +48,7 @@ export const createGroupResolver = authorized< MutationCreateGroupArgs >(async (_, { input }, { uid, log }) => { try { - const userData = await userRepository.findOneBy({ - id: uid, - }) + const userData = await userRepository.findById(uid) if (!userData) { return { errorCodes: [CreateGroupErrorCode.Unauthorized], @@ -107,9 +105,7 @@ export const createGroupResolver = authorized< export const groupsResolver = authorized( async (_, __, { uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [GroupsErrorCode.Unauthorized], diff --git a/packages/api/src/resolvers/save/index.ts b/packages/api/src/resolvers/save/index.ts index 2acf0ce3b..f5508663d 100644 --- a/packages/api/src/resolvers/save/index.ts +++ b/packages/api/src/resolvers/save/index.ts @@ -18,9 +18,9 @@ export const savePageResolver = authorized< SaveSuccess, SaveError, MutationSavePageArgs ->(async (_, { input }, ctx) => { +>(async (_, { input }, { uid }) => { analytics.track({ - userId: ctx.uid, + userId: uid, event: 'link_saved', properties: { url: input.url, @@ -30,9 +30,7 @@ export const savePageResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: ctx.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } @@ -44,11 +42,7 @@ export const saveUrlResolver = authorized< SaveSuccess, SaveError, MutationSaveUrlArgs ->(async (_, { input }, ctx) => { - const { - claims: { uid }, - } = ctx - +>(async (_, { input }, { uid }) => { analytics.track({ userId: uid, event: 'link_saved', @@ -60,9 +54,7 @@ export const saveUrlResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } @@ -74,9 +66,9 @@ export const saveFileResolver = authorized< SaveSuccess, SaveError, MutationSaveFileArgs ->(async (_, { input }, ctx) => { +>(async (_, { input }, { uid }) => { analytics.track({ - userId: ctx.uid, + userId: uid, event: 'link_saved', properties: { url: input.url, @@ -86,9 +78,7 @@ export const saveFileResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: ctx.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } diff --git a/packages/api/src/resolvers/send_install_instructions/index.ts b/packages/api/src/resolvers/send_install_instructions/index.ts index 6435b807b..62d134fbc 100644 --- a/packages/api/src/resolvers/send_install_instructions/index.ts +++ b/packages/api/src/resolvers/send_install_instructions/index.ts @@ -14,11 +14,9 @@ const INSTALL_INSTRUCTIONS_EMAIL_TEMPLATE_ID = export const sendInstallInstructionsResolver = authorized< SendInstallInstructionsSuccess, SendInstallInstructionsError ->(async (_parent, _args, { claims, log }) => { +>(async (_parent, _args, { uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: claims.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SendInstallInstructionsErrorCode.Unauthorized] } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index db249b9b9..7076e8524 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -89,7 +89,8 @@ export const subscriptionsResolver = authorized< } const subscriptions = await queryBuilder - .orderBy(`subscription.${sortBy}`, sortOrder, 'NULLS LAST') + .orderBy('subscription.status', 'ASC') + .addOrderBy(`subscription.${sortBy}`, sortOrder, 'NULLS LAST') .getMany() return { diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index bf99a5c04..38b0111e2 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -52,9 +52,7 @@ export const updateUserResolver = authorized< UpdateUserError, MutationUpdateUserArgs >(async (_, { input: { name, bio } }, { uid, authTrx }) => { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [UpdateUserErrorCode.UserNotFound] } } @@ -92,9 +90,7 @@ export const updateUserProfileResolver = authorized< UpdateUserProfileError, MutationUpdateUserProfileArgs >(async (_, { input: { userId, username, pictureUrl } }, { uid, authTrx }) => { - const user = await userRepository.findOneBy({ - id: userId, - }) + const user = await userRepository.findById(userId) if (!user) { return { errorCodes: [UpdateUserProfileErrorCode.Unauthorized] } } @@ -117,6 +113,7 @@ export const updateUserProfileResolver = authorized< profile: { username: lowerCasedUsername, }, + status: StatusType.Active, }) if (existingUser?.id) { return { @@ -161,6 +158,7 @@ export const googleLoginResolver: ResolverFn< const user = await userRepository.findOneBy({ email, + status: StatusType.Active, }) if (!user?.id) { return { errorCodes: [LoginErrorCode.UserNotFound] } @@ -256,9 +254,7 @@ export const getMeUserResolver: ResolverFn< return undefined } - const user = await userRepository.findOneBy({ - id: claims.uid, - }) + const user = await userRepository.findById(claims.uid) if (!user) { return undefined } @@ -282,12 +278,17 @@ export const getUserResolver: ResolverFn< const userId = id || (username && - (await userRepository.findOneBy({ profile: { username } }))?.id) + ( + await userRepository.findOneBy({ + profile: { username }, + status: StatusType.Active, + }) + )?.id) if (!userId) { return { errorCodes: [UserErrorCode.UserNotFound] } } - const userRecord = await userRepository.findOneBy({ id: userId }) + const userRecord = await userRepository.findById(userId) if (!userRecord) { return { errorCodes: [UserErrorCode.UserNotFound] } } @@ -340,9 +341,7 @@ export const updateEmailResolver = authorized< MutationUpdateEmailArgs >(async (_, { input: { email } }, { authTrx, uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 366c10ada..333391d62 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -421,7 +421,7 @@ export function authRouter() { const { email, password } = req.body try { const user = await userRepository.findByEmail(email.trim()) - if (!user?.id) { + if (!user || user.status === StatusType.Deleted) { return res.redirect( `${env.client.url}/auth/email-login?errorCodes=${LoginErrorCode.UserNotFound}` ) @@ -610,7 +610,7 @@ export function authRouter() { try { const user = await userRepository.findByEmail(email) - if (!user) { + if (!user || user.status === StatusType.Deleted) { return res.redirect(`${env.client.url}/auth/reset-sent`) } diff --git a/packages/api/src/routers/auth/mobile/sign_in.ts b/packages/api/src/routers/auth/mobile/sign_in.ts index 40d4462bb..4af80f0a6 100644 --- a/packages/api/src/routers/auth/mobile/sign_in.ts +++ b/packages/api/src/routers/auth/mobile/sign_in.ts @@ -46,7 +46,7 @@ export async function createMobileEmailSignInResponse( } const user = await userRepository.findByEmail(email.trim()) - if (!user?.id || !user?.password) { + if (!user || !user.password || user.status === StatusType.Deleted) { throw new Error('user not found') } diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts index cc830fc71..8e13ee184 100644 --- a/packages/api/src/routers/user_router.ts +++ b/packages/api/src/routers/user_router.ts @@ -39,7 +39,7 @@ export function userRouter() { return } try { - const user = await userRepository.findOneBy({ id: claims.uid }) + const user = await userRepository.findById(claims.uid) if (!user) { res.status(400).send('Bad Request') return diff --git a/packages/api/src/services/newsletters.ts b/packages/api/src/services/newsletters.ts index fb9e33cca..f492cf4b5 100644 --- a/packages/api/src/services/newsletters.ts +++ b/packages/api/src/services/newsletters.ts @@ -21,10 +21,7 @@ export const createNewsletterEmail = async ( userId: string, confirmationCode?: string ): Promise => { - const user = await userRepository.findOne({ - where: { id: userId }, - relations: ['profile'], - }) + const user = await userRepository.findById(userId) if (!user) { return Promise.reject({ errorCode: CreateNewsletterEmailErrorCode.Unauthorized, diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index 0f14b1658..3165bb8fb 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -42,9 +42,7 @@ export const saveUrlFromEmail = async ( clientRequestId: string, userId: string ): Promise => { - const user = await userRepository.findOneBy({ - id: userId, - }) + const user = await userRepository.findById(userId) if (!user) { return false } diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index 39babc270..870d9ce6b 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -1,4 +1,4 @@ -import { User } from '../entity/user' +import { StatusType, User } from '../entity/user' import { authTrx } from '../repository' import { userRepository } from '../repository/user' @@ -21,5 +21,5 @@ export const updateUser = async (userId: string, update: Partial) => { } export const findUser = async (id: string): Promise => { - return userRepository.findOneBy({ id }) + return userRepository.findOneBy({ id, status: StatusType.Active }) } diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index f8ca6ffa2..af7e40a9a 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -16,6 +16,7 @@ import { ILike } from 'typeorm' import { promisify } from 'util' import { v4 as uuid } from 'uuid' import { Highlight } from '../entity/highlight' +import { StatusType } from '../entity/user' import { env } from '../env' import { PageType, PreparedDocumentInput } from '../generated/graphql' import { userRepository } from '../repository/user' @@ -470,6 +471,7 @@ export const isProbablyArticle = async ( ): Promise => { const user = await userRepository.findOneBy({ email: ILike(email), + status: StatusType.Active, }) return !!user || subject.includes(ARTICLE_PREFIX) } diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 07393fbdb..7a9db963d 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -146,7 +146,7 @@ describe('Subscriptions API', () => { undefined, SubscriptionType.Newsletter ) - const allSubscriptions = [sub5, ...subscriptions] + const allSubscriptions = [...subscriptions, sub5] const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.subscriptions.subscriptions).to.eql( diff --git a/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql new file mode 100755 index 000000000..d8b83487d --- /dev/null +++ b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql @@ -0,0 +1,19 @@ +-- Type: DO +-- Name: add_unique_to_subscription_url +-- Description: Add unique constraint to the url field on the omnivore.subscription table + +BEGIN; + +-- Deleting duplicates first to avoid unique constraint violation +WITH DuplicateCTE AS ( + SELECT user_id, url, + ROW_NUMBER() OVER (PARTITION BY user_id, url ORDER BY (SELECT NULL)) AS RowNum + FROM omnivore.subscriptions + WHERE type = 'RSS' +) +DELETE FROM omnivore.subscriptions + WHERE (user_id, url) IN (SELECT user_id, url FROM DuplicateCTE WHERE RowNum > 1); + +ALTER TABLE omnivore.subscriptions ADD CONSTRAINT subscriptions_user_id_url_key UNIQUE (user_id, url); + +COMMIT; diff --git a/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql b/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql new file mode 100755 index 000000000..72071eab2 --- /dev/null +++ b/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_unique_to_subscription_url +-- Description: Add unique constraint to the url field on the omnivore.subscription table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP CONSTRAINT IF EXISTS subscriptions_user_id_url_key; + +COMMIT; From e4332b74f97b88eda9c702fa34c0f699da679725 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 20:11:51 +0800 Subject: [PATCH 008/115] Add queue manager service --- packages/queue-manager/Dockerfile | 26 +++ packages/queue-manager/mocha-config.json | 5 + packages/queue-manager/package.json | 31 +++ packages/queue-manager/src/index.ts | 190 ++++++++++++++++++ packages/queue-manager/test/babel-register.js | 3 + packages/queue-manager/test/stub.test.ts | 8 + packages/queue-manager/tsconfig.json | 8 + 7 files changed, 271 insertions(+) create mode 100644 packages/queue-manager/Dockerfile create mode 100644 packages/queue-manager/mocha-config.json create mode 100644 packages/queue-manager/package.json create mode 100644 packages/queue-manager/src/index.ts create mode 100644 packages/queue-manager/test/babel-register.js create mode 100644 packages/queue-manager/test/stub.test.ts create mode 100644 packages/queue-manager/tsconfig.json diff --git a/packages/queue-manager/Dockerfile b/packages/queue-manager/Dockerfile new file mode 100644 index 000000000..fe547f483 --- /dev/null +++ b/packages/queue-manager/Dockerfile @@ -0,0 +1,26 @@ +FROM node:14.18-alpine + +# Run everything after as non-privileged user. +WORKDIR /app + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .eslintrc . + +COPY /packages/queue-manager/package.json ./packages/queue-manager/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/queue-manager ./packages/queue-manager +RUN yarn workspace @omnivore/queue-manager build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/queue-manager/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 8080 + +CMD ["yarn", "workspace", "@omnivore/queue-manager", "start"] + diff --git a/packages/queue-manager/mocha-config.json b/packages/queue-manager/mocha-config.json new file mode 100644 index 000000000..44d1d24c1 --- /dev/null +++ b/packages/queue-manager/mocha-config.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "spec": "test/**/*.test.ts", + "require": "test/babel-register.js" + } \ No newline at end of file diff --git a/packages/queue-manager/package.json b/packages/queue-manager/package.json new file mode 100644 index 000000000..69c2630a8 --- /dev/null +++ b/packages/queue-manager/package.json @@ -0,0 +1,31 @@ +{ + "name": "@omnivore/queue-manager", + "version": "1.0.0", + "main": "build/src/index.js", + "files": [ + "build/src" + ], + "license": "Apache-2.0", + "scripts": { + "test": "yarn mocha -r ts-node/register --config mocha-config.json", + "lint": "eslint src --ext ts,js,tsx,jsx", + "compile": "tsc", + "build": "tsc", + "start": "functions-framework --target=queueManager", + "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" + }, + "devDependencies": { + "chai": "^4.3.6", + "eslint-plugin-prettier": "^4.0.0", + "mocha": "^10.0.0" + }, + "dependencies": { + "@google-cloud/functions-framework": "3.1.2", + "@google-cloud/monitoring": "^4.0.0", + "@google-cloud/tasks": "^4.0.0", + "@sentry/serverless": "^6.16.1", + "axios": "^1.4.0", + "dotenv": "^16.0.1", + "jsonwebtoken": "^8.5.1" + } +} \ No newline at end of file diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts new file mode 100644 index 000000000..f2b5791c5 --- /dev/null +++ b/packages/queue-manager/src/index.ts @@ -0,0 +1,190 @@ +import { MetricServiceClient } from '@google-cloud/monitoring' +import { v2beta3 } from '@google-cloud/tasks' +import fetch from 'node-fetch' +import * as dotenv from 'dotenv' + +import * as Sentry from '@sentry/serverless' + +dotenv.config() +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) + +const PROJECT_ID = process.env.PROJECT_ID +const LOCATION = 'us-west2' +const IMPORT_QUEUE_NAME = process.env.IMPORT_QUEUE_NAME +const RSS_QUEUE_NAME = process.env.RSS_FEED_QUEUE_NAME +const QUEUE_NAMES = [IMPORT_QUEUE_NAME, RSS_QUEUE_NAME] +const DISCORD_WEBHOOK_URL = process.env.DISCORD_WEBHOOK_URL +const METRICS_FILTER = `metric.type="appengine.googleapis.com/http/server/response_latencies" metric.labels.response_code="200"` + +if ( + !PROJECT_ID || + !IMPORT_QUEUE_NAME || + !RSS_QUEUE_NAME || + !DISCORD_WEBHOOK_URL +) { + throw new Error('environment not supplied.') +} + +const LATENCY_THRESHOLD = 500 +const RSS_QUEUE_THRESHOLD = 20_000 +const IMPORT_QUEUE_THRESHOLD = 100_000 + +const postToDiscord = async (message: string) => { + console.log('notify message', { message }) + const payload = { + content: message, + } + + try { + const response = await fetch(DISCORD_WEBHOOK_URL, { + method: 'POST', + body: JSON.stringify(payload), + headers: { + 'Content-Type': 'application/json', + }, + }) + + if (!response.ok) { + throw new Error(`Discord response was not ok: ${response.statusText}`) + } + } catch (error) { + console.error('Failed to post message to Discord:', error) + } +} + +const checkShouldPauseQueues = async () => { + const now = Date.now() + const client = new MetricServiceClient() + + // Query for the metrics from the last 5 minutes + const [timeSeries] = await client.listTimeSeries({ + name: client.projectPath(PROJECT_ID), + filter: METRICS_FILTER, + interval: { + startTime: { + seconds: Math.floor(now / 1000 - 5 * 60), + }, + endTime: { + seconds: Math.floor(now / 1000), + }, + }, + aggregation: { + alignmentPeriod: { + seconds: 300, + }, + perSeriesAligner: 'ALIGN_PERCENTILE_95', + }, + }) + + let shouldPauseQueues = false + + for (const ts of timeSeries) { + // We only want to look at the backend service right now + if ( + !ts.resource || + !ts.resource.labels || + !ts.resource.labels['module_id'] || + !ts.resource.labels['module_id'].startsWith('backend') + ) { + continue + } + + if (ts.points && ts.points.length) { + const avgLatency = + ts.points.reduce( + (acc, point) => acc + (point.value?.doubleValue ?? 0), + 0 + ) / ts.points.length + console.log('avgLatency: ', avgLatency) + if (avgLatency > LATENCY_THRESHOLD) { + shouldPauseQueues = true + break + } + } + } + + return shouldPauseQueues +} + +const getQueueTaskCount = async (queueName: string) => { + const cloudTasksClient = new v2beta3.CloudTasksClient() + const queuePath = cloudTasksClient.queuePath(PROJECT_ID, LOCATION, queueName) + const [queue] = await cloudTasksClient.getQueue({ + name: queuePath, + readMask: { paths: ['name', 'stats'] }, + }) + + console.log(' queue.stats', { stats: queue.stats }) + if (Number.isNaN(queue.stats?.tasksCount)) { + return 0 + } + return Number(queue.stats?.tasksCount) +} + +const pauseQueues = async () => { + const cloudTasksClient = new v2beta3.CloudTasksClient() + + await Promise.all([ + cloudTasksClient.pauseQueue({ + name: cloudTasksClient.queuePath(PROJECT_ID, LOCATION, RSS_QUEUE_NAME), + }), + cloudTasksClient.pauseQueue({ + name: cloudTasksClient.queuePath(PROJECT_ID, LOCATION, IMPORT_QUEUE_NAME), + }), + ]) +} + +async function checkMetricsAndPauseQueues() { + if ( + !PROJECT_ID || + !IMPORT_QUEUE_NAME || + !RSS_QUEUE_NAME || + !DISCORD_WEBHOOK_URL + ) { + throw new Error('environment not supplied.') + } + + const shouldPauseQueues = await checkShouldPauseQueues() + + if (shouldPauseQueues) { + const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` + + // Pause the two queues + await pauseQueues() + + // Post to Discord server using webhook + await postToDiscord(message) + } else { + const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + + if (rssQueueCount > RSS_QUEUE_THRESHOLD) { + await postToDiscord( + `The RSS queue has exceeded it's threshold, it has ${rssQueueCount} items in it.` + ) + } + + if (importQueueCount > IMPORT_QUEUE_THRESHOLD) { + await postToDiscord( + `The import queue has exceeded it's threshold, it has ${importQueueCount} items in it.` + ) + } + } +} + +export const queueManager = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + try { + checkMetricsAndPauseQueues() + res.send('ok') + } catch (e) { + console.error('Error while parsing RSS feed', e) + res.status(500).send('INTERNAL_SERVER_ERROR') + } + } +) diff --git a/packages/queue-manager/test/babel-register.js b/packages/queue-manager/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/queue-manager/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/queue-manager/test/stub.test.ts b/packages/queue-manager/test/stub.test.ts new file mode 100644 index 000000000..24ad25c8f --- /dev/null +++ b/packages/queue-manager/test/stub.test.ts @@ -0,0 +1,8 @@ +import 'mocha' +import { expect } from 'chai' + +describe('stub test', () => { + it('should pass', () => { + expect(true).to.be.true + }) +}) diff --git a/packages/queue-manager/tsconfig.json b/packages/queue-manager/tsconfig.json new file mode 100644 index 000000000..7ebe093f6 --- /dev/null +++ b/packages/queue-manager/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./../../tsconfig.json", + "compilerOptions": { + "outDir": "build", + "rootDir": "." + }, + "include": ["src"] +} From 51e2aaa3eae6134d473a72af2c5e801bcacc8807 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 20:29:03 +0800 Subject: [PATCH 009/115] Add node-fetch dependency --- packages/queue-manager/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/queue-manager/package.json b/packages/queue-manager/package.json index 69c2630a8..7c1319a45 100644 --- a/packages/queue-manager/package.json +++ b/packages/queue-manager/package.json @@ -15,6 +15,7 @@ "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" }, "devDependencies": { + "@types/node-fetch": "^2.6.6", "chai": "^4.3.6", "eslint-plugin-prettier": "^4.0.0", "mocha": "^10.0.0" @@ -28,4 +29,4 @@ "dotenv": "^16.0.1", "jsonwebtoken": "^8.5.1" } -} \ No newline at end of file +} From b700c75552f6382537b022f93e071874249f9de2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 16 Oct 2023 22:32:41 +0800 Subject: [PATCH 010/115] Use the GCP project id --- packages/queue-manager/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index f2b5791c5..1e79a304d 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -11,7 +11,7 @@ Sentry.GCPFunction.init({ tracesSampleRate: 0, }) -const PROJECT_ID = process.env.PROJECT_ID +const PROJECT_ID = process.env.GCP_PROJECT_ID const LOCATION = 'us-west2' const IMPORT_QUEUE_NAME = process.env.IMPORT_QUEUE_NAME const RSS_QUEUE_NAME = process.env.RSS_FEED_QUEUE_NAME From ab0fc1087d5cd2429d476eed61793126ab16eeda Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 12:02:44 +0800 Subject: [PATCH 011/115] update tsv only if necessary --- ...7.do.alter_library_item_tsv_update_trigger.sql | 15 +++++++++++++++ ...undo.alter_library_item_tsv_update_trigger.sql | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100755 packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql create mode 100755 packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql diff --git a/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql b/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql new file mode 100755 index 000000000..aa7d266d8 --- /dev/null +++ b/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql @@ -0,0 +1,15 @@ +-- Type: DO +-- Name: alter_library_item_tsv_update_trigger +-- Description: Alter library_item_tsv_update trigger on omnivore.library_item table to add conditions + +BEGIN; + +DROP TRIGGER IF EXISTS library_item_tsv_update ON omnivore.library_item; + +CREATE TRIGGER library_item_tsv_update + BEFORE INSERT OR UPDATE OF readable_content, site_name, title, author, description, note, highlight_annotations + ON omnivore.library_item + FOR EACH ROW + EXECUTE PROCEDURE update_library_item_tsv(); + +COMMIT; diff --git a/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql b/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql new file mode 100755 index 000000000..cb367e944 --- /dev/null +++ b/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql @@ -0,0 +1,15 @@ +-- Type: UNDO +-- Name: alter_library_item_tsv_update_trigger +-- Description: Alter library_item_tsv_update trigger on omnivore.library_item table to add conditions + +BEGIN; + +DROP TRIGGER IF EXISTS library_item_tsv_update ON omnivore.library_item; + +CREATE TRIGGER library_item_tsv_update + BEFORE INSERT OR UPDATE + ON omnivore.library_item + FOR EACH ROW + EXECUTE PROCEDURE update_library_item_tsv(); + +COMMIT; From 1f600c272e7208c11822bba61118ad2fbe4d86fe Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 13:36:47 +0800 Subject: [PATCH 012/115] Better handling of network failures for labels/subscriptions/searches in the left menu --- .../templates/homeFeed/LibraryFilterMenu.tsx | 41 +++++++++++-------- .../networking/queries/useGetLabelsQuery.tsx | 5 ++- .../queries/useGetSavedSearchQuery.tsx | 3 ++ .../queries/useGetSubscriptionsQuery.tsx | 3 ++ 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx index 11adec862..fba2015c9 100644 --- a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx +++ b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx @@ -48,32 +48,39 @@ export function LibraryFilterMenu(props: LibraryFilterMenuProps): JSX.Element { isSessionStorage: false, initialValue: [], }) - const { labels: networkLabels, isLoading: labelsLoading } = - useGetLabelsQuery() - const { savedSearches: networkSearches, isLoading: searchesLoading } = - useGetSavedSearchQuery() - const { - subscriptions: networkSubscriptions, - isLoading: subscriptionsLoading, - } = useGetSubscriptionsQuery() + const labelsResponse = useGetLabelsQuery() + const searchesResponse = useGetSavedSearchQuery() + const subscriptionsResponse = useGetSubscriptionsQuery() useEffect(() => { - if (!labelsLoading) { - setLabels(networkLabels) + if ( + !labelsResponse.error && + !labelsResponse.isLoading && + labelsResponse.labels + ) { + setLabels(labelsResponse.labels) } - }, [setLabels, networkLabels, labelsLoading]) + }, [setLabels, labelsResponse]) useEffect(() => { - if (!subscriptionsLoading) { - setSubscriptions(networkSubscriptions) + if ( + !subscriptionsResponse.error && + !subscriptionsResponse.isLoading && + subscriptionsResponse.subscriptions + ) { + setSubscriptions(subscriptionsResponse.subscriptions) } - }, [setSubscriptions, networkSubscriptions, subscriptionsLoading]) + }, [setSubscriptions, subscriptionsResponse]) useEffect(() => { - if (!searchesLoading) { - setSavedSearches(networkSearches ?? []) + if ( + !searchesResponse.error && + !searchesResponse.isLoading && + searchesResponse.savedSearches + ) { + setSavedSearches(searchesResponse.savedSearches) } - }, [setSavedSearches, networkSearches, searchesLoading]) + }, [setSavedSearches, searchesResponse]) return ( <> diff --git a/packages/web/lib/networking/queries/useGetLabelsQuery.tsx b/packages/web/lib/networking/queries/useGetLabelsQuery.tsx index d6fefad8e..abd4295fd 100644 --- a/packages/web/lib/networking/queries/useGetLabelsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetLabelsQuery.tsx @@ -4,6 +4,7 @@ import { Label, labelFragment } from '../fragments/labelFragment' import { publicGqlFetcher } from '../networkHelpers' type LabelsQueryResponse = { + error: any isLoading: boolean isValidating: boolean labels: Label[] @@ -38,10 +39,11 @@ export function useGetLabelsQuery(): LabelsQueryResponse { const { data, error, mutate, isValidating } = useSWR(query, publicGqlFetcher) try { - if (data) { + if (data && !error) { const result = data as LabelsResponseData const labels = result.labels?.labels as Label[] return { + error, isLoading: !error && !data, isValidating, labels, @@ -54,6 +56,7 @@ export function useGetLabelsQuery(): LabelsQueryResponse { console.log('error', error) } return { + error, isLoading: !error && !data, isValidating: false, labels: [], diff --git a/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx b/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx index 4dde76f89..f6a76f386 100644 --- a/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx +++ b/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx @@ -7,6 +7,7 @@ import { } from '../fragments/savedSearchFragment' type SavedSearchResponse = { + error: any savedSearches?: SavedSearch[] savedSearchErrors?: unknown isLoading: boolean @@ -39,6 +40,7 @@ export function useGetSavedSearchQuery(): SavedSearchResponse { const { filters } = data as SavedSearchResponseData return { + error, savedSearches: filters?.filters ?? [], savedSearchErrors: error ?? {}, isLoading: false, @@ -46,6 +48,7 @@ export function useGetSavedSearchQuery(): SavedSearchResponse { } return { + error, savedSearches: [], savedSearchErrors: null, isLoading: !error && !data, diff --git a/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx b/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx index 4c0a26d17..ababa6fdb 100644 --- a/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx @@ -25,6 +25,7 @@ export type Subscription = { } type SubscriptionsQueryResponse = { + error: any isLoading: boolean isValidating: boolean subscriptions: Subscription[] @@ -85,6 +86,7 @@ export function useGetSubscriptionsQuery( const result = data as SubscriptionsResponseData const subscriptions = result.subscriptions.subscriptions as Subscription[] return { + error, isLoading: !error && !data, isValidating, subscriptions, @@ -97,6 +99,7 @@ export function useGetSubscriptionsQuery( console.log('error', error) } return { + error, isLoading: !error && !data, isValidating: true, subscriptions: [], From ffa476a109ee44726596125603bd4355b25d9cba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Oct 2023 05:44:10 +0000 Subject: [PATCH 013/115] Bump @babel/traverse from 7.14.7 to 7.23.2 in /pkg/admin Bumps [@babel/traverse](https://github.com/babel/babel/tree/HEAD/packages/babel-traverse) from 7.14.7 to 7.23.2. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.23.2/packages/babel-traverse) --- updated-dependencies: - dependency-name: "@babel/traverse" dependency-type: indirect ... Signed-off-by: dependabot[bot] --- pkg/admin/yarn.lock | 144 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 132 insertions(+), 12 deletions(-) diff --git a/pkg/admin/yarn.lock b/pkg/admin/yarn.lock index eb2623a42..0b67d4ca9 100644 --- a/pkg/admin/yarn.lock +++ b/pkg/admin/yarn.lock @@ -33,6 +33,14 @@ dependencies: "@babel/highlight" "^7.14.5" +"@babel/code-frame@^7.22.13": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== + dependencies: + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" + "@babel/compat-data@^7.13.11", "@babel/compat-data@^7.14.5", "@babel/compat-data@^7.14.7": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.7.tgz#7b047d7a3a89a67d2258dc61f604f098f1bc7e08" @@ -68,6 +76,16 @@ jsesc "^2.5.1" source-map "^0.5.0" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.0.0", "@babel/helper-annotate-as-pure@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.14.5.tgz#7bf478ec3b71726d56a8ca5775b046fc29879e61" @@ -127,6 +145,11 @@ resolve "^1.14.2" semver "^6.1.2" +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + "@babel/helper-explode-assignable-expression@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.14.5.tgz#8aa72e708205c7bb643e45c73b4386cdf2a1f645" @@ -143,6 +166,14 @@ "@babel/template" "^7.14.5" "@babel/types" "^7.14.5" +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== + dependencies: + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" + "@babel/helper-get-function-arity@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.14.5.tgz#25fbfa579b0937eee1f3b805ece4ce398c431815" @@ -157,6 +188,13 @@ dependencies: "@babel/types" "^7.14.5" +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-member-expression-to-functions@^7.14.5": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.14.7.tgz#97e56244beb94211fe277bd818e3a329c66f7970" @@ -237,11 +275,28 @@ dependencies: "@babel/types" "^7.14.5" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + +"@babel/helper-string-parser@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" + integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== + "@babel/helper-validator-identifier@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz#d0f0e277c512e0c938277faa85a3968c9a44c0e8" integrity sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-option@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.14.5.tgz#6e72a1fff18d5dfcb878e1e62f1a021c4b72d5a3" @@ -275,11 +330,25 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.10.2", "@babel/parser@^7.12.5", "@babel/parser@^7.14.5", "@babel/parser@^7.14.6", "@babel/parser@^7.14.7": +"@babel/highlight@^7.22.13": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" + js-tokens "^4.0.0" + +"@babel/parser@^7.10.2", "@babel/parser@^7.12.5", "@babel/parser@^7.14.5", "@babel/parser@^7.14.6": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.7.tgz#6099720c8839ca865a2637e6c85852ead0bdb595" integrity sha512-X67Z5y+VBJuHB/RjwECp8kSl5uYi0BvRbNeWqkaJCVh+LiTPl19WBUfG627psSgp9rSf6ojuXghQM3ha6qHHdA== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.14.5.tgz#4b467302e1548ed3b1be43beae2cc9cf45e0bb7e" @@ -977,18 +1046,28 @@ "@babel/parser" "^7.14.5" "@babel/types" "^7.14.5" -"@babel/traverse@^7.12.5", "@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.4.5": - version "7.14.7" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.7.tgz#64007c9774cfdc3abd23b0780bc18a3ce3631753" - integrity sha512-9vDr5NzHu27wgwejuKL7kIOm4bwEtaPQ4Z6cpCmjSuaRqpH/7xc4qcGEscwMqlkwgcXl6MvqoAjZkQ24uSdIZQ== +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== dependencies: - "@babel/code-frame" "^7.14.5" - "@babel/generator" "^7.14.5" - "@babel/helper-function-name" "^7.14.5" - "@babel/helper-hoist-variables" "^7.14.5" - "@babel/helper-split-export-declaration" "^7.14.5" - "@babel/parser" "^7.14.7" - "@babel/types" "^7.14.5" + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.12.5", "@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.4.5": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -1000,6 +1079,15 @@ "@babel/helper-validator-identifier" "^7.14.5" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@carbon/icon-helpers@^10.19.0": version "10.19.0" resolved "https://registry.yarnpkg.com/@carbon/icon-helpers/-/icon-helpers-10.19.0.tgz#f6b608b181b4ca4aeeadac72ec11b7cf530b4d1c" @@ -1131,6 +1219,38 @@ gud "^1.0.0" warning "^4.0.3" +"@jridgewell/gen-mapping@^0.3.2": + version "0.3.3" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098" + integrity sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ== + dependencies: + "@jridgewell/set-array" "^1.0.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.9" + +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" + integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + +"@jridgewell/set-array@^1.0.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" + integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== + +"@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9": + version "0.3.19" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz#f8a3249862f91be48d3127c3cfe992f79b4b8811" + integrity sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" From d141fea9777838299a012ce95eb34cc6dea6c680 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 14:18:45 +0800 Subject: [PATCH 014/115] Handle cases where fetching queue counts fails --- packages/queue-manager/src/index.ts | 50 +++++++++++++++++------------ 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 1e79a304d..190903a96 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -150,29 +150,37 @@ async function checkMetricsAndPauseQueues() { const shouldPauseQueues = await checkShouldPauseQueues() if (shouldPauseQueues) { - const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) - const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) - const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` - - // Pause the two queues - await pauseQueues() - - // Post to Discord server using webhook - await postToDiscord(message) - } else { - const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) - const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) - - if (rssQueueCount > RSS_QUEUE_THRESHOLD) { - await postToDiscord( - `The RSS queue has exceeded it's threshold, it has ${rssQueueCount} items in it.` - ) + let rssQueueCount: number | string = 'unknown' + let importQueueCount: number | string = 'unknown' + try { + rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + } catch (err) { + console.log('error fetching queue counts', err) } - if (importQueueCount > IMPORT_QUEUE_THRESHOLD) { - await postToDiscord( - `The import queue has exceeded it's threshold, it has ${importQueueCount} items in it.` - ) + const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` + + await pauseQueues() + await postToDiscord(message) + } else { + try { + const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + + if (rssQueueCount > RSS_QUEUE_THRESHOLD) { + await postToDiscord( + `The RSS queue has exceeded it's threshold, it has ${rssQueueCount} items in it.` + ) + } + + if (importQueueCount > IMPORT_QUEUE_THRESHOLD) { + await postToDiscord( + `The import queue has exceeded it's threshold, it has ${importQueueCount} items in it.` + ) + } + } catch (err) { + console.log('error getting queue counts') } } } From a6d3bb22b11e192d50aa7a87d8cf9e5ade3b4811 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 14:46:57 +0800 Subject: [PATCH 015/115] Add a check param so we can avoid checks on container startup while deploying --- packages/queue-manager/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 190903a96..4d560b337 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -188,7 +188,9 @@ async function checkMetricsAndPauseQueues() { export const queueManager = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { try { - checkMetricsAndPauseQueues() + if (req.query['check']) { + checkMetricsAndPauseQueues() + } res.send('ok') } catch (e) { console.error('Error while parsing RSS feed', e) From eb667e04b92246d0819937140fa581f27741e1e8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 15:09:16 +0800 Subject: [PATCH 016/115] update reading progress in db --- packages/api/src/resolvers/article/index.ts | 62 +++++++----------- packages/api/src/services/library_item.ts | 69 +++++++++++++++++++++ packages/api/test/resolvers/article.test.ts | 1 - 3 files changed, 90 insertions(+), 42 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index bf26872c5..bf7dc89f9 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -62,11 +62,11 @@ import { } from '../../services/labels' import { createLibraryItem, - findLibraryItemById, findLibraryItemByUrl, findLibraryItemsByPrefix, searchLibraryItems, updateLibraryItem, + updateLibraryItemReadingProgress, updateLibraryItems, } from '../../services/library_item' import { parsedContentToLibraryItem } from '../../services/save_page' @@ -572,14 +572,8 @@ export const saveArticleReadingProgressResolver = authorized< readingProgressTopPercent, }, }, - { uid, pubsub } + { log, pubsub, uid } ) => { - const libraryItem = await findLibraryItemById(id, uid) - - if (!libraryItem) { - return { errorCodes: [SaveArticleReadingProgressErrorCode.NotFound] } - } - if ( readingProgressPercent < 0 || readingProgressPercent > 100 || @@ -590,40 +584,26 @@ export const saveArticleReadingProgressResolver = authorized< ) { return { errorCodes: [SaveArticleReadingProgressErrorCode.BadData] } } - // If we have a top percent, we only save it if it's greater than the current top percent - // or set to zero if the top percent is zero. - const readingProgressTopPercentToSave = readingProgressTopPercent - ? Math.max( - readingProgressTopPercent, - libraryItem.readingProgressTopPercent || 0 - ) - : readingProgressTopPercent === 0 - ? 0 - : undefined - // If setting to zero we accept the update, otherwise we require it - // be greater than the current reading progress. - const updatedPart: QueryDeepPartialEntity = { - readingProgressBottomPercent: - readingProgressPercent === 0 - ? 0 - : Math.max( - readingProgressPercent, - libraryItem.readingProgressBottomPercent - ), - readingProgressHighestReadAnchor: - readingProgressAnchorIndex === 0 - ? 0 - : Math.max( - readingProgressAnchorIndex || 0, - libraryItem.readingProgressHighestReadAnchor - ), - readingProgressTopPercent: readingProgressTopPercentToSave, - readAt: new Date(), - } - const updatedItem = await updateLibraryItem(id, updatedPart, uid, pubsub) + try { + const updatedItem = await updateLibraryItemReadingProgress( + id, + uid, + readingProgressPercent, + readingProgressTopPercent, + readingProgressAnchorIndex, + pubsub + ) + if (!updatedItem) { + return { errorCodes: [SaveArticleReadingProgressErrorCode.BadData] } + } - return { - updatedArticle: libraryItemToArticle(updatedItem), + return { + updatedArticle: libraryItemToArticle(updatedItem), + } + } catch (error) { + log.error('saveArticleReadingProgressResolver error', error) + + return { errorCodes: [SaveArticleReadingProgressErrorCode.Unauthorized] } } } ) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 07be9f885..8dab74fc5 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -438,6 +438,75 @@ export const updateLibraryItem = async ( return updatedLibraryItem } +export const updateLibraryItemReadingProgress = async ( + id: string, + userId: string, + bottomPercent: number, + topPercent: number | null = null, + anchorIndex: number | null = null, + pubsub = createPubSubClient() +): Promise => { + // If we have a top percent, we only save it if it's greater than the current top percent + // or set to zero if the top percent is zero. + const result = (await authTrx( + async (tx) => + tx.getRepository(LibraryItem).query( + ` + UPDATE omnivore.library_item + SET reading_progress_top_percent = CASE + WHEN reading_progress_top_percent < $2 THEN $2 + WHEN $2 = 0 THEN 0 + ELSE reading_progress_top_percent + END, + reading_progress_bottom_percent = CASE + WHEN reading_progress_bottom_percent < $3 THEN $3 + WHEN $3 = 0 THEN 0 + ELSE reading_progress_bottom_percent + END, + reading_progress_highest_read_anchor = CASE + WHEN reading_progress_top_percent < $4 THEN $4 + WHEN $4 = 0 THEN 0 + ELSE reading_progress_highest_read_anchor + END, + read_at = now() + WHERE id = $1 AND ( + (reading_progress_top_percent < $2 OR $2 = 0) OR + (reading_progress_bottom_percent < $3 OR $3 = 0) OR + (reading_progress_highest_read_anchor < $4 OR $4 = 0) + ) + RETURNING + id, + reading_progress_top_percent as "readingProgressTopPercent", + reading_progress_bottom_percent as "readingProgressBottomPercent", + reading_progress_highest_read_anchor as "readingProgressHighestReadAnchor", + read_at as "readAt" + `, + [id, topPercent, bottomPercent, anchorIndex] + ), + undefined, + userId + )) as [LibraryItem[], number] + if (result[1] === 0) { + return null + } + + const updatedItem = result[0][0] + await pubsub.entityUpdated>( + EntityType.PAGE, + { + id, + readingProgressBottomPercent: updatedItem.readingProgressBottomPercent, + readingProgressTopPercent: updatedItem.readingProgressTopPercent, + readingProgressHighestReadAnchor: + updatedItem.readingProgressHighestReadAnchor, + readAt: updatedItem.readAt, + }, + userId + ) + + return updatedItem +} + export const createLibraryItems = async ( libraryItems: DeepPartial[], userId: string diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index e3b13c0d1..083bd389e 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -706,7 +706,6 @@ describe('Article API', () => { ).to.eq(75) // Now try to set to a lower value (50), value should not be updated - // refresh index to ensure the reading progress is updated const secondQuery = saveArticleReadingProgressQuery(itemId, 50) const secondRes = await graphqlRequest(secondQuery, authToken).expect(200) expect( From 97cb8d3af7fb2694cd422f7002f409be7c316a21 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 15:53:05 +0800 Subject: [PATCH 017/115] Bump test limit --- packages/queue-manager/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 4d560b337..3ddc6a62e 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -30,7 +30,7 @@ if ( const LATENCY_THRESHOLD = 500 const RSS_QUEUE_THRESHOLD = 20_000 -const IMPORT_QUEUE_THRESHOLD = 100_000 +const IMPORT_QUEUE_THRESHOLD = 200_000 const postToDiscord = async (message: string) => { console.log('notify message', { message }) From d450bdd8e65f1c23faa9dcd30ea27f2fd2d24a8f Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 16:03:38 +0800 Subject: [PATCH 018/115] Linting improvements --- packages/queue-manager/.eslintignore | 2 ++ packages/queue-manager/.eslintrc | 6 ++++++ packages/queue-manager/src/index.ts | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 packages/queue-manager/.eslintignore create mode 100644 packages/queue-manager/.eslintrc diff --git a/packages/queue-manager/.eslintignore b/packages/queue-manager/.eslintignore new file mode 100644 index 000000000..b38db2f29 --- /dev/null +++ b/packages/queue-manager/.eslintignore @@ -0,0 +1,2 @@ +node_modules/ +build/ diff --git a/packages/queue-manager/.eslintrc b/packages/queue-manager/.eslintrc new file mode 100644 index 000000000..e006282a6 --- /dev/null +++ b/packages/queue-manager/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "tsconfig.json" + } +} \ No newline at end of file diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 3ddc6a62e..f26246914 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -189,7 +189,7 @@ export const queueManager = Sentry.GCPFunction.wrapHttpFunction( async (req, res) => { try { if (req.query['check']) { - checkMetricsAndPauseQueues() + await checkMetricsAndPauseQueues() } res.send('ok') } catch (e) { From 6e79f6c67fdb288fa1aa9dd73f6b0d73edd983be Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 16:41:14 +0800 Subject: [PATCH 019/115] update sql to deduplicate rss subscriptions --- packages/api/src/resolvers/subscriptions/index.ts | 1 - .../db/migrations/0136.do.add_unique_to_subscription_url.sql | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 7076e8524..74375441d 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -185,7 +185,6 @@ export const subscribeResolver = authorized< url: input.url || undefined, name: input.name || undefined, user: { id: uid }, - status: SubscriptionStatus.Active, type: input.subscriptionType || SubscriptionType.Rss, // default to rss }) ) diff --git a/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql index d8b83487d..2bfea4029 100755 --- a/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql +++ b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql @@ -6,13 +6,12 @@ BEGIN; -- Deleting duplicates first to avoid unique constraint violation WITH DuplicateCTE AS ( - SELECT user_id, url, - ROW_NUMBER() OVER (PARTITION BY user_id, url ORDER BY (SELECT NULL)) AS RowNum + SELECT id, ROW_NUMBER() OVER (PARTITION BY user_id, url ORDER BY status, last_fetched_at DESC NULLS LAST) AS row_number FROM omnivore.subscriptions WHERE type = 'RSS' ) DELETE FROM omnivore.subscriptions - WHERE (user_id, url) IN (SELECT user_id, url FROM DuplicateCTE WHERE RowNum > 1); + WHERE id IN (SELECT id FROM DuplicateCTE WHERE row_number > 1); ALTER TABLE omnivore.subscriptions ADD CONSTRAINT subscriptions_user_id_url_key UNIQUE (user_id, url); From 700fd73dbbd9894b44d8be3d7bbbfdd9edd7854f Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 17:07:46 +0800 Subject: [PATCH 020/115] disallow social login if account is deleted --- packages/api/src/routers/auth/apple_auth.ts | 2 ++ packages/api/src/routers/auth/google_auth.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/api/src/routers/auth/apple_auth.ts b/packages/api/src/routers/auth/apple_auth.ts index 925e7d37e..99590ff4d 100644 --- a/packages/api/src/routers/auth/apple_auth.ts +++ b/packages/api/src/routers/auth/apple_auth.ts @@ -15,6 +15,7 @@ import { suggestedUsername, } from './jwt_helpers' import { analytics } from '../../utils/analytics' +import { StatusType } from '../../entity/user' const appleBaseURL = 'https://appleid.apple.com' const audienceName = 'app.omnivore.app' @@ -122,6 +123,7 @@ export async function handleAppleWebAuth( const user = await userRepository.findOneBy({ sourceUserId: decodedTokenResult.sourceUserId, source: 'APPLE', + status: StatusType.Active, }) const userId = user?.id diff --git a/packages/api/src/routers/auth/google_auth.ts b/packages/api/src/routers/auth/google_auth.ts index 781c45dcd..1108b6f23 100644 --- a/packages/api/src/routers/auth/google_auth.ts +++ b/packages/api/src/routers/auth/google_auth.ts @@ -1,6 +1,7 @@ import { google, oauth2_v2 as oauthV2 } from 'googleapis' import { OAuth2Client } from 'googleapis-common' import url from 'url' +import { StatusType } from '../../entity/user' import { env, homePageURL } from '../../env' import { LoginErrorCode } from '../../generated/graphql' import { userRepository } from '../../repository/user' @@ -130,6 +131,7 @@ export async function handleGoogleWebAuth( const user = await userRepository.findOneBy({ email, source: 'GOOGLE', + status: StatusType.Active, }) const userId = user?.id From 5f11bf1fded163844ff5ff1b84f7e464e2eae34b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 17:12:26 +0800 Subject: [PATCH 021/115] remove elasticsearch dep from packages/db --- packages/db/package.json | 1 - yarn.lock | 23 +---------------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/packages/db/package.json b/packages/db/package.json index bf7cfd766..fce36c4af 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -9,7 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "@elastic/elasticsearch": "~7.12.0", "dotenv": "^8.2.0", "pg": "^8.3.0", "postgrator": "^4.1.1", diff --git a/yarn.lock b/yarn.lock index 3125e7f03..90f5c0f62 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2228,17 +2228,6 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw== -"@elastic/elasticsearch@~7.12.0": - version "7.12.0" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.12.0.tgz#dbb51a2841f644b670a56d8c15899e860928856f" - integrity sha512-GquUEytCijFRPEk3DKkkDdyhspB3qbucVQOwih9uNyz3iz804I+nGBUsFo2LwVvLQmQfEM0IY2+yoYfEz5wMug== - dependencies: - debug "^4.3.1" - hpagent "^0.1.1" - ms "^2.1.3" - pump "^3.0.0" - secure-json-parse "^2.3.1" - "@emotion/cache@^10.0.27": version "10.0.29" resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-10.0.29.tgz#87e7e64f412c060102d589fe7c6dc042e6f9d1e0" @@ -15835,11 +15824,6 @@ hpack.js@^2.1.6: readable-stream "^2.0.1" wbuf "^1.1.0" -hpagent@^0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9" - integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ== - html-encoding-sniffer@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" @@ -20177,7 +20161,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -24468,11 +24452,6 @@ search-query-parser@^1.6.0: resolved "https://registry.yarnpkg.com/search-query-parser/-/search-query-parser-1.6.0.tgz#d69ade33f3685cae25613a70189b7b18970b46f1" integrity sha512-bhf+phLlKF38nuniwLcVHWPArHGdzenlPhPi955CR3vm1QQifXIuPHwAffhjapojdVVzmv4hgIJ6NOX1d/w+Uw== -secure-json-parse@^2.3.1: - version "2.4.0" - resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85" - integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg== - selderee@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7" From 7eadda02e8f0735bea52849057095ba7d02aa34c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:18:58 +0800 Subject: [PATCH 022/115] Post the avg latency in the update message --- packages/queue-manager/src/index.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index f26246914..47383ba58 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -98,15 +98,13 @@ const checkShouldPauseQueues = async () => { (acc, point) => acc + (point.value?.doubleValue ?? 0), 0 ) / ts.points.length - console.log('avgLatency: ', avgLatency) if (avgLatency > LATENCY_THRESHOLD) { - shouldPauseQueues = true - break + return [true, avgLatency] } } } - return shouldPauseQueues + return [false, 0] } const getQueueTaskCount = async (queueName: string) => { @@ -147,7 +145,7 @@ async function checkMetricsAndPauseQueues() { throw new Error('environment not supplied.') } - const shouldPauseQueues = await checkShouldPauseQueues() + const [shouldPauseQueues, avgLatency] = await checkShouldPauseQueues() if (shouldPauseQueues) { let rssQueueCount: number | string = 'unknown' @@ -159,7 +157,7 @@ async function checkMetricsAndPauseQueues() { console.log('error fetching queue counts', err) } - const message = `Both queues have been paused due to API latency threshold exceedance.\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` + const message = `Both queues have been paused due to API latency threshold exceedance (${avgLatency}).\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` await pauseQueues() await postToDiscord(message) From d8a5ed42e8d52c422fb8e67fdd41a208e36685d4 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:28:56 +0800 Subject: [PATCH 023/115] Remove unused var --- packages/queue-manager/src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 47383ba58..27d13e22b 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -79,8 +79,6 @@ const checkShouldPauseQueues = async () => { }, }) - let shouldPauseQueues = false - for (const ts of timeSeries) { // We only want to look at the backend service right now if ( From fd57e075d7ca112dbdb883b30845df11d4f7a083 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 17 Oct 2023 18:30:57 +0800 Subject: [PATCH 024/115] Clean up return signature --- packages/queue-manager/src/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 27d13e22b..37124ce7b 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -97,12 +97,12 @@ const checkShouldPauseQueues = async () => { 0 ) / ts.points.length if (avgLatency > LATENCY_THRESHOLD) { - return [true, avgLatency] + return { shouldPauseQueues: true, avgLatency: avgLatency } } } } - return [false, 0] + return { shouldPauseQueues: false, avgLatency: 0 } } const getQueueTaskCount = async (queueName: string) => { @@ -143,7 +143,7 @@ async function checkMetricsAndPauseQueues() { throw new Error('environment not supplied.') } - const [shouldPauseQueues, avgLatency] = await checkShouldPauseQueues() + const { shouldPauseQueues, avgLatency } = await checkShouldPauseQueues() if (shouldPauseQueues) { let rssQueueCount: number | string = 'unknown' From 7f61acc907d6f9f3466650543d586ab649a1bfa2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 19:40:29 +0800 Subject: [PATCH 025/115] skip old feed if the pubDate is before last_fetched timestamp --- packages/rss-handler/src/index.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 9c37e22f9..60e789e73 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -122,6 +122,7 @@ const parser = new Parser({ maxRedirects: 10, customFields: { item: [['link', 'links', { keepArray: true }], 'published', 'updated'], + feed: ['dc:date', 'lastBuildDate', 'pubDate'], }, headers: { // some rss feeds require user agent @@ -198,7 +199,16 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( // fetch feed let itemCount = 0 const feed = await parser.parseURL(feedUrl) - console.log('Fetched feed', feed.title, new Date()) + console.log('Fetched feed', feed, new Date()) + + const feedPubDate = (feed['dc:date'] || + feed.pubDate || + feed.lastBuildDate) as string | undefined + console.log('Feed pub date', feedPubDate) + if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { + console.log('Skipping old feed', feedPubDate) + return res.send('ok') + } // save each item in the feed for (const item of feed.items) { From 2c830176685473e16539340962eba1084e349e6f Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 18 Oct 2023 03:00:40 +0900 Subject: [PATCH 026/115] Fix typo in ContentLoading.swift propogate -> propagate --- .../Sources/Services/DataService/ContentLoading.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift index f32e5005c..d276c40e1 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift @@ -231,7 +231,7 @@ extension DataService { _ = try await saveURL(id: id, url: url) } } catch { - // We don't propogate these errors, we just let it pass through so + // We don't propagate these errors, we just let it pass through so // the user can attempt to fetch content again. print("Error syncUnsyncedArticleContent", error) } From d315418171f68d849434221183d4c3893a522762 Mon Sep 17 00:00:00 2001 From: Paul Mullins Date: Tue, 17 Oct 2023 18:49:37 -0400 Subject: [PATCH 027/115] Fix password in readme, password is demo_password not demo, see packages/db/setup.sh for reference --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 54bf3b2b4..698528626 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ Open and confirm Omnivore is running ### 3. Login with the test account -During database setup docker compose creates an account `demo@omnivore.app`, password: `demo`. +During database setup docker compose creates an account `demo@omnivore.app`, password: `demo_password`. Go to in your browser and choose `Continue with Email` to login. From 1053c9b53a719627c62b7dccb1d811641427b8c3 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 09:11:11 +0800 Subject: [PATCH 028/115] Bump threshold for testing --- packages/queue-manager/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts index 37124ce7b..9574d0684 100644 --- a/packages/queue-manager/src/index.ts +++ b/packages/queue-manager/src/index.ts @@ -30,7 +30,7 @@ if ( const LATENCY_THRESHOLD = 500 const RSS_QUEUE_THRESHOLD = 20_000 -const IMPORT_QUEUE_THRESHOLD = 200_000 +const IMPORT_QUEUE_THRESHOLD = 250_000 const postToDiscord = async (message: string) => { console.log('notify message', { message }) From 25c0051fd4ab807d2ad770d7164ed9f951252497 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:14:35 +0800 Subject: [PATCH 029/115] Only use signed URLs for PDF attachments --- packages/api/src/resolvers/article/index.ts | 6 +----- packages/api/src/routers/svc/email_attachment.ts | 6 +----- packages/api/src/utils/uploads.ts | 16 ---------------- 3 files changed, 2 insertions(+), 26 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index bf7dc89f9..bd016a345 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -98,10 +98,7 @@ import { parsePreparedContent, } from '../../utils/parser' import { parseSearchQuery, sortParamsToSort } from '../../utils/search' -import { - getStorageFileDetails, - makeStorageFilePublic, -} from '../../utils/uploads' +import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' export enum ArticleFormat { @@ -310,7 +307,6 @@ export const createArticleResolver = authorized< pubsub ) } - await makeStorageFilePublic(uploadFileData.id, uploadFileData.fileName) } let libraryItemToReturn: LibraryItem diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index b9bf3c77c..926317f4e 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -143,11 +143,7 @@ export function emailAttachmentRouter() { return res.status(400).send('BAD REQUEST') } - const uploadFileUrlOverride = await makeStorageFilePublic( - uploadFileData.id, - uploadFileData.fileName - ) - + const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFileId}/${uploadFile.fileName}` const uploadFileHash = uploadFileDetails.md5Hash const itemType = uploadFile.contentType === 'application/pdf' diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 34bfb5854..1cfeb1ff8 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -81,22 +81,6 @@ export const generateDownloadSignedUrl = async ( return url } -export const makeStorageFilePublic = async ( - id: string, - fileName: string -): Promise => { - // if (env.dev.isLocal) { - // return 'http://localhost:3000/public/' + id + '/' + fileName - // } - - // Makes the file public - const filePathName = generateUploadFilePathName(id, fileName) - await storage.bucket(bucketName).file(filePathName).makePublic() - - const fileObj = storage.bucket(bucketName).file(filePathName) - return fileObj.publicUrl() -} - export const getStorageFileDetails = async ( id: string, fileName: string From b6ce4f39b1de907de48d08b7ad514dde6c8e0780 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:15:49 +0800 Subject: [PATCH 030/115] Fix imports --- packages/api/src/routers/svc/email_attachment.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index 926317f4e..096359673 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -24,7 +24,6 @@ import { generateUploadFilePathName, generateUploadSignedUrl, getStorageFileDetails, - makeStorageFilePublic, } from '../../utils/uploads' export function emailAttachmentRouter() { From 4220922ff80db3b57ea41f7ccbc5ed224ec86c44 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:23:26 +0800 Subject: [PATCH 031/115] Preserve original URL if possible --- packages/api/src/resolvers/upload_files/index.ts | 6 +----- packages/api/src/utils/uploads.ts | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 36169b5c4..643d758d4 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -110,13 +110,9 @@ export const uploadFileRequestResolver = authorized< input.contentType ) - const publicUrl = getFilePublicUrl(uploadFilePathName) - - // If this is a file URL, we swap in the GCS public URL if (isFileUrl(input.url)) { await authTrx(async (tx) => { await tx.getRepository(UploadFile).update(uploadFileId, { - url: publicUrl, status: UploadFileStatus.Initialized, }) }) @@ -142,7 +138,7 @@ export const uploadFileRequestResolver = authorized< const uploadFileId = uploadFileData.id const item = await createLibraryItem( { - originalUrl: isFileUrl(input.url) ? publicUrl : input.url, + originalUrl: input.url, id: input.clientRequestId || undefined, user: { id: uid }, title, diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 1cfeb1ff8..2d31b525f 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -34,10 +34,6 @@ const storage = env.fileUpload?.gcsUploadSAKeyFilePath : new Storage() const bucketName = env.fileUpload.gcsUploadBucket -export const getFilePublicUrl = (filePathName: string): string => { - return storage.bucket(bucketName).file(filePathName).publicUrl() -} - export const countOfFilesWithPrefix = async (prefix: string) => { const [files] = await storage.bucket(bucketName).getFiles({ prefix }) return files.length From 78843fff14558a12912b85fe402269eabcb9e746 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:27:09 +0800 Subject: [PATCH 032/115] Remove import --- packages/api/src/resolvers/upload_files/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 643d758d4..58eca5370 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -24,7 +24,6 @@ import { contentReaderForLibraryItem, generateUploadFilePathName, generateUploadSignedUrl, - getFilePublicUrl, } from '../../utils/uploads' const isFileUrl = (url: string): boolean => { From b88c64424f48954e37f3477db6e2ae5fd3fd41a3 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:37:21 +0800 Subject: [PATCH 033/115] Special handling of file URLs --- packages/api/src/resolvers/upload_files/index.ts | 3 +++ packages/api/src/routers/svc/email_attachment.ts | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 58eca5370..93e622ea0 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -109,9 +109,12 @@ export const uploadFileRequestResolver = authorized< input.contentType ) + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` if (isFileUrl(input.url)) { await authTrx(async (tx) => { await tx.getRepository(UploadFile).update(uploadFileId, { + url: attachmentUrl, status: UploadFileStatus.Initialized, }) }) diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index 096359673..6c127135b 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -142,7 +142,12 @@ export function emailAttachmentRouter() { return res.status(400).send('BAD REQUEST') } - const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFileId}/${uploadFile.fileName}` + const uploadFilePathName = generateUploadFilePathName( + uploadFileId, + uploadFile.fileName + ) + + const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFilePathName}` const uploadFileHash = uploadFileDetails.md5Hash const itemType = uploadFile.contentType === 'application/pdf' From 71aaef3628b8650da3ef56641b4c8cb79d5b286d Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 10:43:53 +0800 Subject: [PATCH 034/115] Update attachment URL --- packages/api/src/resolvers/upload_files/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 93e622ea0..9ea214584 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -140,8 +140,8 @@ export const uploadFileRequestResolver = authorized< const uploadFileId = uploadFileData.id const item = await createLibraryItem( { - originalUrl: input.url, id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, user: { id: uid }, title, readableContent: '', From 0874f6ff21fb0489ad222ca65e9412dc3b360aca Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 22:12:38 +0800 Subject: [PATCH 035/115] log x-forwarded-for header for debugging purpose --- packages/api/src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index cc757acc3..c36d25148 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -72,6 +72,7 @@ export const createApp = (): { } }, keyGenerator: (req) => { + console.log('x-forwarded-for header', req.header('x-forwarded-for')) return getTokenByRequest(req) || req.ip }, // skip preflight requests and test requests From 91076014cded16d70449a50dba807a748ca881c2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 17 Oct 2023 22:47:41 +0800 Subject: [PATCH 036/115] log req.ip --- packages/api/src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index c36d25148..14004e642 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -73,6 +73,7 @@ export const createApp = (): { }, keyGenerator: (req) => { console.log('x-forwarded-for header', req.header('x-forwarded-for')) + console.log('ip', req.ip) return getTokenByRequest(req) || req.ip }, // skip preflight requests and test requests From 32ba744d20802e24a2ab0fdedb1c05267026ab20 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 18 Oct 2023 10:59:31 +0800 Subject: [PATCH 037/115] test trust proxy=1 --- packages/api/src/server.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 14004e642..0c9456934 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -57,6 +57,7 @@ export const createApp = (): { app.use(cookieParser()) app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) + app.set('trust proxy', 1) const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute @@ -72,8 +73,8 @@ export const createApp = (): { } }, keyGenerator: (req) => { - console.log('x-forwarded-for header', req.header('x-forwarded-for')) - console.log('ip', req.ip) + console.log('x-forwarded-for header:', req.header('x-forwarded-for')) + console.log('ip:', req.ip) return getTokenByRequest(req) || req.ip }, // skip preflight requests and test requests From 506004d410de99fcc1bd0b1100ff22416d533be2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 18 Oct 2023 11:58:15 +0800 Subject: [PATCH 038/115] test trust proxy=2 --- packages/api/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 0c9456934..dbad3ec94 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -57,7 +57,7 @@ export const createApp = (): { app.use(cookieParser()) app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) - app.set('trust proxy', 1) + app.set('trust proxy', 2) const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute From 988b20e30caf8d60390aaff239957c1b04fd9dd3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 18 Oct 2023 12:30:27 +0800 Subject: [PATCH 039/115] test trust proxy=true --- packages/api/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index dbad3ec94..00c7d0e09 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -57,7 +57,7 @@ export const createApp = (): { app.use(cookieParser()) app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) - app.set('trust proxy', 2) + app.set('trust proxy', true) const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute From 46313b14ba1fd4d5dcf03c9a08acb104b17fcbc5 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Wed, 18 Oct 2023 13:06:33 +0800 Subject: [PATCH 040/115] set trust proxy = true if set in env var --- packages/api/src/server.ts | 6 +++--- packages/api/src/util.ts | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 00c7d0e09..9e41af64d 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -57,7 +57,9 @@ export const createApp = (): { app.use(cookieParser()) app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) - app.set('trust proxy', true) + + // set to true if behind a reverse proxy/load balancer + app.set('trust proxy', env.server.trustProxy) const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute @@ -73,8 +75,6 @@ export const createApp = (): { } }, keyGenerator: (req) => { - console.log('x-forwarded-for header:', req.header('x-forwarded-for')) - console.log('ip:', req.ip) return getTokenByRequest(req) || req.ip }, // skip preflight requests and test requests diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 71df1cbc5..5764307a2 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -21,6 +21,7 @@ interface BackendEnv { gateway_url: string apiEnv: string instanceId: string + trustProxy: boolean } client: { url: string @@ -159,6 +160,7 @@ const nullableEnvVars = [ 'RSS_FEED_TASK_HANDLER_URL', 'SENDGRID_VERIFICATION_TEMPLATE_ID', 'REMINDER_TASK_HANDLER_URL', + 'TRUST_PROXY', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -207,6 +209,7 @@ export function getEnv(): BackendEnv { apiEnv: parse('API_ENV'), instanceId: parse('GAE_INSTANCE') || `x${os.userInfo().username}_${os.hostname()}`, + trustProxy: parse('TRUST_PROXY') === 'true', } const client = { url: parse('CLIENT_URL'), From cf101c6d18e39f252d7b2b0355803201ca0b96be Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:26:03 +0800 Subject: [PATCH 041/115] Cache and check feed checksums to reduce fetching --- packages/api/src/entity/subscription.ts | 3 + packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + .../api/src/resolvers/subscriptions/index.ts | 1 + packages/api/src/routers/svc/rss_feed.ts | 2 +- packages/api/src/schema.ts | 1 + packages/api/src/utils/createTask.ts | 1 + ...do.add_checksum_to_subscriptions_table.sql | 7 ++ ...do.add_checksum_to_subscriptions_table.sql | 9 ++ packages/rss-handler/src/index.ts | 84 +++++++++++++------ 10 files changed, 85 insertions(+), 25 deletions(-) create mode 100755 packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql create mode 100755 packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index 90f7d593b..b63400d56 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -59,6 +59,9 @@ export class Subscription { @Column('timestamp', { nullable: true }) lastFetchedAt?: Date | null + @Column('text', { nullable: true }) + lastFetchedChecksum?: string | null + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) createdAt!: Date diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index f0f41ff68..8ceb89481 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2978,6 +2978,7 @@ export type UpdateSubscriptionInput = { description?: InputMaybe; id: Scalars['ID']; lastFetchedAt?: InputMaybe; + lastfetchedChecksum?: InputMaybe; name?: InputMaybe; status?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 3ef33f886..39a6c00e7 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2391,6 +2391,7 @@ input UpdateSubscriptionInput { description: String id: ID! lastFetchedAt: Date + lastfetchedChecksum: String name: String status: SubscriptionStatus } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 74375441d..a1069423b 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -290,6 +290,7 @@ export const updateSubscriptionResolver = authorized< lastFetchedAt: input.lastFetchedAt ? new Date(input.lastFetchedAt) : undefined, + lastFetchedChecksum: input.lastfetchedChecksum, status: input.status || undefined, }) diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 0834e5025..c06ec9f71 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -24,7 +24,7 @@ export function rssFeedRouter() { // get all active rss feed subscriptions const subscriptions = await getRepository(Subscription).find({ - select: ['id', 'url', 'user', 'lastFetchedAt'], + select: ['id', 'url', 'user', 'lastFetchedAt', 'lastFetchedChecksum'], where: { type: SubscriptionType.Rss, status: SubscriptionStatus.Active, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e3b2b3442..6d78471cb 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2548,6 +2548,7 @@ const schema = gql` name: String description: String lastFetchedAt: Date + lastfetchedChecksum: String status: SubscriptionStatus } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index bb3457e00..483921bc9 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -601,6 +601,7 @@ export const enqueueRssFeedFetch = async ( subscriptionId: rssFeedSubscription.id, feedUrl: rssFeedSubscription.url, lastFetchedAt: rssFeedSubscription.lastFetchedAt?.getTime() || 0, // unix timestamp in milliseconds + lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, } const headers = { diff --git a/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql new file mode 100755 index 000000000..5f704a666 --- /dev/null +++ b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql @@ -0,0 +1,7 @@ +-- Type: DO +-- Name: add_checksum_to_subscriptions_table +-- Description: Add a last fetched checksum field to the subscriptions table + +BEGIN; + +COMMIT; diff --git a/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql b/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql new file mode 100755 index 000000000..e138d501a --- /dev/null +++ b/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_checksum_to_subscriptions_table +-- Description: Add a last fetched checksum field to the subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP COLUMN last_fetched_checksum ; + +COMMIT; diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 60e789e73..4edd58709 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -1,5 +1,6 @@ import * as Sentry from '@sentry/serverless' import axios from 'axios' +import crypto from 'crypto' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import * as jwt from 'jsonwebtoken' import Parser, { Item } from 'rss-parser' @@ -10,6 +11,7 @@ interface RssFeedRequest { subscriptionId: string feedUrl: string lastFetchedAt: number // unix timestamp in milliseconds + lastFetchedChecksum: string | undefined } // link can be a string or an object @@ -21,10 +23,42 @@ function isRssFeedRequest(body: any): body is RssFeedRequest { ) } +type FeedFetchResult = { + url: string + content: string + checksum: string +} + +async function fetchAndChecksum(url: string): Promise { + try { + // Fetch the content from the URL + const response = await axios.get(url, { + responseType: 'arraybuffer', + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + Accept: + 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', + }, + }) + + // Create a sha256 hash of the content + const hash = crypto.createHash('sha256') + hash.update(response.data) + + return { url, content: response.data, checksum: hash.digest('hex') } + } catch (error) { + throw new Error( + `Failed to fetch or hash content from ${url}. Error: ${error}` + ) + } +} + const sendUpdateSubscriptionMutation = async ( userId: string, subscriptionId: string, - lastFetchedAt: Date + lastFetchedAt: Date, + lastFetchedChecksum: string ) => { const JWT_SECRET = process.env.JWT_SECRET const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT @@ -51,6 +85,7 @@ const sendUpdateSubscriptionMutation = async ( input: { id: subscriptionId, lastFetchedAt, + lastFetchedChecksum, }, }, }) @@ -121,15 +156,12 @@ const parser = new Parser({ timeout: 60000, // 60 seconds maxRedirects: 10, customFields: { - item: [['link', 'links', { keepArray: true }], 'published', 'updated'], - feed: ['dc:date', 'lastBuildDate', 'pubDate'], - }, - headers: { - // some rss feeds require user agent - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - Accept: - 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', + item: [ + ['link', 'links', { keepArray: true }], + 'published', + 'updated', + 'created', + ], }, }) @@ -190,31 +222,34 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( return res.status(400).send('INVALID_REQUEST_BODY') } - const { feedUrl, subscriptionId, lastFetchedAt } = req.body + const { feedUrl, subscriptionId, lastFetchedAt, lastFetchedChecksum } = + req.body console.log('Processing feed', feedUrl, lastFetchedAt) let lastItemFetchedAt: Date | null = null let lastValidItem: Item | null = null + let updatedLastFetchedChecksum: string | null + + let fetchResult = await fetchAndChecksum(feedUrl) + if (fetchResult.checksum === lastFetchedChecksum) { + console.log('feed has not been updated', feedUrl, lastFetchedChecksum) + return res.status(200) + } + updatedLastFetchedChecksum = fetchResult.checksum // fetch feed let itemCount = 0 - const feed = await parser.parseURL(feedUrl) - console.log('Fetched feed', feed, new Date()) - - const feedPubDate = (feed['dc:date'] || - feed.pubDate || - feed.lastBuildDate) as string | undefined - console.log('Feed pub date', feedPubDate) - if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { - console.log('Skipping old feed', feedPubDate) - return res.send('ok') - } + const feed = await parser.parseString(fetchResult.content) + console.log('Fetched feed', feed.title, new Date()) // save each item in the feed for (const item of feed.items) { // use published or updated if isoDate is not available for atom feeds item.isoDate = - item.isoDate || (item.published as string) || (item.updated as string) + item.isoDate || + (item.published as string) || + (item.updated as string) || + (item.created as string) console.log('Processing feed item', item.links, item.isoDate) if (!item.links || item.links.length === 0) { @@ -299,7 +334,8 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( const updatedSubscription = await sendUpdateSubscriptionMutation( userId, subscriptionId, - lastItemFetchedAt + lastItemFetchedAt, + updatedLastFetchedChecksum ) console.log('Updated subscription', updatedSubscription) From f14fc034fe95ed20631c98b8e2c56691d759d8b7 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:28:22 +0800 Subject: [PATCH 042/115] Add last fetched column to the database --- .../migrations/0138.do.add_checksum_to_subscriptions_table.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql index 5f704a666..dfc37edfe 100755 --- a/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql +++ b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql @@ -4,4 +4,6 @@ BEGIN; +ALTER TABLE omnivore.subscriptions ADD COLUMN last_fetched_checksum TEXT ; + COMMIT; From 78484136454b64f3f3ace2b21e48e26532920e78 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:34:20 +0800 Subject: [PATCH 043/115] Update schema --- packages/api/src/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 6d78471cb..2cc4e8b58 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2548,7 +2548,7 @@ const schema = gql` name: String description: String lastFetchedAt: Date - lastfetchedChecksum: String + lastFetchedChecksum: String status: SubscriptionStatus } From d592dc44c6a6b931e5757dfddde1ae998aa55b83 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:34:40 +0800 Subject: [PATCH 044/115] Add back missing pubdate check --- packages/rss-handler/src/index.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 4edd58709..048592c27 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -242,6 +242,15 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( const feed = await parser.parseString(fetchResult.content) console.log('Fetched feed', feed.title, new Date()) + const feedPubDate = (feed['dc:date'] || + feed.pubDate || + feed.lastBuildDate) as string | undefined + console.log('Feed pub date', feedPubDate) + if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { + console.log('Skipping old feed', feedPubDate) + return res.send('ok') + } + // save each item in the feed for (const item of feed.items) { // use published or updated if isoDate is not available for atom feeds From 921a46a13ae7828823f7513093dfae1d025681f2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 16:51:23 +0800 Subject: [PATCH 045/115] Linting fixes --- packages/rss-handler/src/index.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 048592c27..5d1a3c654 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -31,7 +31,6 @@ type FeedFetchResult = { async function fetchAndChecksum(url: string): Promise { try { - // Fetch the content from the URL const response = await axios.get(url, { responseType: 'arraybuffer', headers: { @@ -42,15 +41,15 @@ async function fetchAndChecksum(url: string): Promise { }, }) - // Create a sha256 hash of the content const hash = crypto.createHash('sha256') - hash.update(response.data) + hash.update(response.data as Buffer) - return { url, content: response.data, checksum: hash.digest('hex') } + const dataStr = (response.data as Buffer).toString() + + return { url, content: dataStr, checksum: hash.digest('hex') } } catch (error) { - throw new Error( - `Failed to fetch or hash content from ${url}. Error: ${error}` - ) + console.log(error) + throw new Error(`Failed to fetch or hash content from ${url}.`) } } @@ -228,14 +227,13 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( let lastItemFetchedAt: Date | null = null let lastValidItem: Item | null = null - let updatedLastFetchedChecksum: string | null - let fetchResult = await fetchAndChecksum(feedUrl) + const fetchResult = await fetchAndChecksum(feedUrl) if (fetchResult.checksum === lastFetchedChecksum) { console.log('feed has not been updated', feedUrl, lastFetchedChecksum) return res.status(200) } - updatedLastFetchedChecksum = fetchResult.checksum + const updatedLastFetchedChecksum = fetchResult.checksum // fetch feed let itemCount = 0 From 479fc8fb7d96bdec37505dcea683593e6f051de2 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 17:11:02 +0800 Subject: [PATCH 046/115] Add a test for checksumming --- packages/rss-handler/package.json | 3 ++- packages/rss-handler/src/index.ts | 8 +------- packages/rss-handler/test/checksum.test.ts | 14 ++++++++++++++ packages/rss-handler/test/stub.test.ts | 8 -------- 4 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 packages/rss-handler/test/checksum.test.ts delete mode 100644 packages/rss-handler/test/stub.test.ts diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json index 2a6d40851..0db5d93fd 100644 --- a/packages/rss-handler/package.json +++ b/packages/rss-handler/package.json @@ -17,7 +17,8 @@ "devDependencies": { "chai": "^4.3.6", "eslint-plugin-prettier": "^4.0.0", - "mocha": "^10.0.0" + "mocha": "^10.0.0", + "nock": "^13.3.4" }, "dependencies": { "@google-cloud/functions-framework": "3.1.2", diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 5d1a3c654..2b1bdf1fc 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -23,13 +23,7 @@ function isRssFeedRequest(body: any): body is RssFeedRequest { ) } -type FeedFetchResult = { - url: string - content: string - checksum: string -} - -async function fetchAndChecksum(url: string): Promise { +export const fetchAndChecksum = async (url: string) => { try { const response = await axios.get(url, { responseType: 'arraybuffer', diff --git a/packages/rss-handler/test/checksum.test.ts b/packages/rss-handler/test/checksum.test.ts new file mode 100644 index 000000000..24eee7a0b --- /dev/null +++ b/packages/rss-handler/test/checksum.test.ts @@ -0,0 +1,14 @@ +import 'mocha' +import nock from 'nock' +import { expect } from 'chai' +import { fetchAndChecksum } from '../src/index' + +describe('fetchAndChecksum', () => { + it('should hash the content available', async () => { + nock('https://fake.com', {}).get('/rss.xml').reply(200, 'i am some content') + const result = await fetchAndChecksum('https://fake.com/rss.xml') + expect(result.checksum).to.eq( + 'd6bc10faec048d999d0cf4b2f7103d84557fb9cd94c3bccd17884b1288949375' + ) + }) +}) diff --git a/packages/rss-handler/test/stub.test.ts b/packages/rss-handler/test/stub.test.ts deleted file mode 100644 index 24ad25c8f..000000000 --- a/packages/rss-handler/test/stub.test.ts +++ /dev/null @@ -1,8 +0,0 @@ -import 'mocha' -import { expect } from 'chai' - -describe('stub test', () => { - it('should pass', () => { - expect(true).to.be.true - }) -}) From d2c0efc540142d46c4df639b946bddfd03331d61 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 18:22:22 +0800 Subject: [PATCH 047/115] add some debug --- packages/rss-handler/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2b1bdf1fc..a61bdfbec 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -83,6 +83,8 @@ const sendUpdateSubscriptionMutation = async ( }, }) + console.log('sending', data) + const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string try { const response = await axios.post( From fbaaaeca7c67714d9ce7189994ad8fd6d7ae8e7b Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 18:48:18 +0800 Subject: [PATCH 048/115] Add some debugging --- packages/api/src/resolvers/subscriptions/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index a1069423b..8cc6c4d01 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -290,7 +290,7 @@ export const updateSubscriptionResolver = authorized< lastFetchedAt: input.lastFetchedAt ? new Date(input.lastFetchedAt) : undefined, - lastFetchedChecksum: input.lastfetchedChecksum, + lastFetchedChecksum: input.lastfetchedChecksum || undefined, status: input.status || undefined, }) @@ -300,6 +300,8 @@ export const updateSubscriptionResolver = authorized< }) }) + console.log('updatedSubscription', updatedSubscription) + return { subscription: updatedSubscription, } From 5c576347a8878701c7debe114419ca6173e2f724 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 19:26:45 +0800 Subject: [PATCH 049/115] MOre debug --- packages/api/src/resolvers/subscriptions/index.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 8cc6c4d01..6b415e4d4 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -282,8 +282,7 @@ export const updateSubscriptionResolver = authorized< const updatedSubscription = await authTrx(async (t) => { const repo = t.getRepository(Subscription) - // update subscription - await t.getRepository(Subscription).save({ + const dict = { id: input.id, name: input.name || undefined, description: input.description || undefined, @@ -292,7 +291,10 @@ export const updateSubscriptionResolver = authorized< : undefined, lastFetchedChecksum: input.lastfetchedChecksum || undefined, status: input.status || undefined, - }) + } + console.log('saving dict:', JSON.stringify(dict)) + // update subscription + await t.getRepository(Subscription).save(dict) return repo.findOneByOrFail({ id: input.id, From 0fcc7096aa9598bacb88eb2ce22fae4a4e9f905d Mon Sep 17 00:00:00 2001 From: Surav Shrestha Date: Wed, 18 Oct 2023 17:33:22 +0545 Subject: [PATCH 050/115] docs: fix typo in packages/puppeteer-parse/README.md --- packages/puppeteer-parse/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/README.md b/packages/puppeteer-parse/README.md index 6155c45b0..373c57353 100644 --- a/packages/puppeteer-parse/README.md +++ b/packages/puppeteer-parse/README.md @@ -1,3 +1,3 @@ # Puppeteer parsing function handler -This workspace is used to provide the module for the app to hande requests for the article parsing via Puppeteer. +This workspace is used to provide the module for the app to handle requests for the article parsing via Puppeteer. From 78a642db6d0142628e387376b2ed08ce17d32ee1 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 19:53:26 +0800 Subject: [PATCH 051/115] Update type signature --- packages/api/src/generated/graphql.ts | 2 +- packages/api/src/generated/schema.graphql | 2 +- packages/api/src/resolvers/subscriptions/index.ts | 10 ++++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 8ceb89481..0ad412b4f 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2978,7 +2978,7 @@ export type UpdateSubscriptionInput = { description?: InputMaybe; id: Scalars['ID']; lastFetchedAt?: InputMaybe; - lastfetchedChecksum?: InputMaybe; + lastFetchedChecksum?: InputMaybe; name?: InputMaybe; status?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 39a6c00e7..af9064bee 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2391,7 +2391,7 @@ input UpdateSubscriptionInput { description: String id: ID! lastFetchedAt: Date - lastfetchedChecksum: String + lastFetchedChecksum: String name: String status: SubscriptionStatus } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 6b415e4d4..50fdcb839 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -282,19 +282,17 @@ export const updateSubscriptionResolver = authorized< const updatedSubscription = await authTrx(async (t) => { const repo = t.getRepository(Subscription) - const dict = { + // update subscription + await t.getRepository(Subscription).save({ id: input.id, name: input.name || undefined, description: input.description || undefined, lastFetchedAt: input.lastFetchedAt ? new Date(input.lastFetchedAt) : undefined, - lastFetchedChecksum: input.lastfetchedChecksum || undefined, + lastFetchedChecksum: input.lastFetchedChecksum || undefined, status: input.status || undefined, - } - console.log('saving dict:', JSON.stringify(dict)) - // update subscription - await t.getRepository(Subscription).save(dict) + }) return repo.findOneByOrFail({ id: input.id, From 3a4547a6afbf391e6ae00dca92271623819a9d4a Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 21:01:01 +0800 Subject: [PATCH 052/115] Remove debug --- packages/rss-handler/src/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index a61bdfbec..2b1bdf1fc 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -83,8 +83,6 @@ const sendUpdateSubscriptionMutation = async ( }, }) - console.log('sending', data) - const auth = (await signToken({ uid: userId }, JWT_SECRET)) as string try { const response = await axios.post( From c93e61ec247745ac44fb0356e2db878de3afcada Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Wed, 18 Oct 2023 21:30:23 +0800 Subject: [PATCH 053/115] Remove debug --- packages/api/src/resolvers/subscriptions/index.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 50fdcb839..430347654 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -300,8 +300,6 @@ export const updateSubscriptionResolver = authorized< }) }) - console.log('updatedSubscription', updatedSubscription) - return { subscription: updatedSubscription, } From 4b171c0657ffc85e1a43d69279b341b582173a10 Mon Sep 17 00:00:00 2001 From: Surav Shrestha Date: Wed, 18 Oct 2023 20:35:35 +0545 Subject: [PATCH 054/115] docs: fix typos in packages/content-fetch/README.md --- packages/content-fetch/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/content-fetch/README.md b/packages/content-fetch/README.md index 501c004ee..c72e5720f 100644 --- a/packages/content-fetch/README.md +++ b/packages/content-fetch/README.md @@ -1,6 +1,6 @@ # Puppeteer parsing function handler -This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. +This workspace is used to provide the GCF for the app to handle requests for the article parsing via Puppeteer. ## Using locally @@ -8,7 +8,7 @@ Copy .env.example file to .env file: `cp .env.example .env` Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). -After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) +After this, you should be able to access the function on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) ## Deployment From 699e8077dcfd738f0fa5c252b1165c68eb259053 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 19 Oct 2023 09:57:28 +0800 Subject: [PATCH 055/115] Add feed custom fields, set timeout/redirects --- packages/rss-handler/src/index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2b1bdf1fc..ca6afb04b 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -27,6 +27,8 @@ export const fetchAndChecksum = async (url: string) => { try { const response = await axios.get(url, { responseType: 'arraybuffer', + timeout: 60_000, + maxRedirects: 10, headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', @@ -146,8 +148,6 @@ Sentry.GCPFunction.init({ const signToken = promisify(jwt.sign) const parser = new Parser({ - timeout: 60000, // 60 seconds - maxRedirects: 10, customFields: { item: [ ['link', 'links', { keepArray: true }], @@ -155,6 +155,7 @@ const parser = new Parser({ 'updated', 'created', ], + feed: ['dc:date', 'lastBuildDate', 'pubDate'], }, }) From 1733cf756dc96b6e6f6a7a48e7b556dce28f0dd5 Mon Sep 17 00:00:00 2001 From: h3n4l Date: Thu, 19 Oct 2023 11:42:42 +0800 Subject: [PATCH 056/115] docs: add native ios app source code link in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 698528626..111cdb065 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ We built Omnivore because we love reading and we want it to be more social. Join - Add newsletter articles via email (with substack support!) - PDF support - [Web app](https://omnivore.app/) written in Node.js and TypeScript -- [Native iOS app](https://omnivore.app/install/ios) +- [Native iOS app](https://omnivore.app/install/ios) ([source](https://github.com/omnivore-app/omnivore/tree/main/apple)) - [Android app](https://omnivore.app/install/android) ([source](https://github.com/omnivore-app/omnivore/tree/main/android/Omnivore)) - Progressive web app for Android users - Browser extensions for [Chrome](https://omnivore.app/install/chrome), [Safari](https://omnivore.app/install/safari), [Firefox](https://omnivore.app/install/firefox), and [Edge](https://omnivore.app/install/edge) From 88a46e6ee67c2582aa341f47fc5f1e76b2baf212 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 19 Oct 2023 14:18:01 +0800 Subject: [PATCH 057/115] Send ok when checksum has not been updated --- packages/rss-handler/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index ca6afb04b..8b3021659 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -226,7 +226,7 @@ export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( const fetchResult = await fetchAndChecksum(feedUrl) if (fetchResult.checksum === lastFetchedChecksum) { console.log('feed has not been updated', feedUrl, lastFetchedChecksum) - return res.status(200) + return res.send('ok') } const updatedLastFetchedChecksum = fetchResult.checksum From 18aece84e78d1769fadfc67f632d3030fa25535b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 10:24:42 +0800 Subject: [PATCH 058/115] add scheduledAt to subscriptions --- packages/api/src/entity/subscription.ts | 3 +++ .../0138.do.add_scheduled_at_to_subscription.sql | 9 +++++++++ .../0138.undo.add_scheduled_at_to_subscription.sql | 9 +++++++++ 3 files changed, 21 insertions(+) create mode 100755 packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql create mode 100755 packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index b63400d56..fea63d890 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -67,4 +67,7 @@ export class Subscription { @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) updatedAt!: Date + + @Column('timestamp', { nullable: true }) + scheduledAt?: Date | null } diff --git a/packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql new file mode 100755 index 000000000..c45201a06 --- /dev/null +++ b/packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: add_scheduled_at_to_subscription +-- Description: Add scheduled_at field to omnivore.subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions ADD COLUMN scheduled_at timestamptz; + +COMMIT; diff --git a/packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql new file mode 100755 index 000000000..090f63ac4 --- /dev/null +++ b/packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_scheduled_at_to_subscription +-- Description: Add scheduled_at field to omnivore.subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP COLUMN IF EXISTS scheduled_at; + +COMMIT; From 5283a8490873aca5d636f6b1bbce77bb98c3f203 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 11:35:58 +0800 Subject: [PATCH 059/115] add scheduledAt to the task payload --- packages/api/src/services/subscriptions.ts | 16 ++- packages/api/src/utils/createTask.ts | 1 + .../api/test/resolvers/subscriptions.test.ts | 98 ++++++++++++++++++- 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 8aaea89b8..3a30469a8 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -1,4 +1,5 @@ import axios from 'axios' +import { DeleteResult } from 'typeorm' import { appDataSource } from '../data_source' import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' @@ -189,7 +190,8 @@ export const createSubscription = async ( newsletterEmail?: NewsletterEmail, status = SubscriptionStatus.Active, unsubscribeMailTo?: string, - subscriptionType = SubscriptionType.Newsletter + subscriptionType = SubscriptionType.Newsletter, + url?: string ): Promise => { return getRepository(Subscription).save({ user: { id: userId }, @@ -199,5 +201,17 @@ export const createSubscription = async ( unsubscribeMailTo, lastFetchedAt: new Date(), type: subscriptionType, + url, }) } + +export const deleteSubscription = async ( + id: string, + userId: string +): Promise => { + return authTrx( + (tx) => tx.getRepository(Subscription).delete(id), + undefined, + userId + ) +} diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 483921bc9..f7cc4c829 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -602,6 +602,7 @@ export const enqueueRssFeedFetch = async ( feedUrl: rssFeedSubscription.url, lastFetchedAt: rssFeedSubscription.lastFetchedAt?.getTime() || 0, // unix timestamp in milliseconds lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, + scheduledAt: new Date().getTime(), // unix timestamp in milliseconds } const headers = { diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 7a9db963d..22ee107b1 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -1,5 +1,6 @@ import chai, { expect } from 'chai' import 'mocha' +import Parser from 'rss-parser' import sinon from 'sinon' import sinonChai from 'sinon-chai' import { NewsletterEmail } from '../../src/entity/newsletter_email' @@ -9,13 +10,13 @@ import { SubscriptionStatus, SubscriptionType, } from '../../src/generated/graphql' -import { - createSubscription, - unsubscribe, - UNSUBSCRIBE_EMAIL_TEXT -} from '../../src/services/subscriptions' import { getRepository } from '../../src/repository' import { createNewsletterEmail } from '../../src/services/newsletters' +import { + createSubscription, + deleteSubscription, + UNSUBSCRIBE_EMAIL_TEXT, +} from '../../src/services/subscriptions' import { deleteUser } from '../../src/services/user' import * as sendEmail from '../../src/utils/sendEmail' import { createTestUser } from '../db' @@ -347,4 +348,91 @@ describe('Subscriptions API', () => { await getRepository(Subscription).remove(subscription) }) }) + + describe('Subscribe API', () => { + const query = ( + name: string | null, + url: string | null, + subscriptionType: SubscriptionType | null + ) => ` + mutation { + subscribe(input: { + name: ${name ? `"${name}"` : null} + url: ${url ? `"${url}"` : null} + subscriptionType: ${subscriptionType} + }) { + ... on SubscribeSuccess { + subscriptions { + id + } + } + ... on SubscribeError { + errorCodes + } + } + } + ` + + context('when subscribing to a rss feed', () => { + const url = 'https://www.omnivore.app/rss' + const subscriptionType = SubscriptionType.Rss + + before(async () => { + // fake rss parser + sinon.replace(Parser.prototype, 'parseURL', sinon.fake.resolves({ + title: 'RSS Feed', + description: 'RSS Feed Description', + })) + }) + + after(() => { + sinon.restore() + }) + + context('when the user is subscribed to the feed', () => { + let existingSubscription: Subscription + + before(async () => { + existingSubscription = await createSubscription( + user.id, + 'RSS Feed', + undefined, + SubscriptionStatus.Active, + url, + subscriptionType, + url + ) + }) + + after(async () => { + await deleteSubscription(existingSubscription.id, user.id) + }) + + it('returns an error', async () => { + const res = await graphqlRequest( + query(null, url, subscriptionType), + authToken + ).expect(200) + expect(res.body.data.subscribe.errorCodes).to.eql([ + 'ALREADY_SUBSCRIBED', + ]) + }) + }) + + it('creates a rss subscription', async () => { + const res = await graphqlRequest( + query(null, url, subscriptionType), + authToken + ).expect(200) + expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) + expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') + + // clean up + await deleteSubscription( + res.body.data.subscribe.subscriptions[0].id, + user.id + ) + }) + }) + }) }) From f50d1c9cb92fb8ffecf2a3031d5595c6982fa6ef Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 13:41:09 +0800 Subject: [PATCH 060/115] Group RSS subscriptions by feed URL and create cloud task for the groups --- .../api/src/resolvers/subscriptions/index.ts | 12 ++- packages/api/src/routers/svc/rss_feed.ts | 40 ++++--- packages/api/src/services/subscriptions.ts | 8 +- packages/api/src/utils/createTask.ts | 47 ++++---- packages/api/test/routers/rss_feed.test.ts | 101 ++++++++++++++++++ 5 files changed, 173 insertions(+), 35 deletions(-) create mode 100644 packages/api/test/routers/rss_feed.test.ts diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 430347654..7bf145452 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -235,11 +235,19 @@ export const subscribeResolver = authorized< } } + const newSubscription = newSubscriptions[0] + // create a cloud task to fetch rss feed item for the new subscription - await enqueueRssFeedFetch(uid, newSubscriptions[0]) + await enqueueRssFeedFetch({ + user_ids: [uid], + url: input.url, + subscription_ids: [newSubscription.id], + scheduled_timestamps: [null], + last_fetched_timestamps: [null], + }) return { - subscriptions: newSubscriptions, + subscriptions: [newSubscription], } } diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index c06ec9f71..93077f6e4 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -4,7 +4,10 @@ import { Subscription } from '../../entity/subscription' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' import { readPushSubscription } from '../../pubsub' import { getRepository } from '../../repository' -import { enqueueRssFeedFetch } from '../../utils/createTask' +import { + enqueueRssFeedFetch, + RssSubscriptionGroup, +} from '../../utils/createTask' import { logger } from '../../utils/logger' export function rssFeedRouter() { @@ -22,21 +25,34 @@ export function rssFeedRouter() { return res.status(200).send('Expired') } - // get all active rss feed subscriptions - const subscriptions = await getRepository(Subscription).find({ - select: ['id', 'url', 'user', 'lastFetchedAt', 'lastFetchedChecksum'], - where: { - type: SubscriptionType.Rss, - status: SubscriptionStatus.Active, - }, - relations: ['user'], - }) + // get active rss feed subscriptions scheduled for fetch and group by feed url + const subscriptionGroups = (await getRepository(Subscription).query( + ` + SELECT + url, + ARRAY_AGG(id) AS subscription_ids, + ARRAY_AGG(user_id) AS user_ids, + ARRAY_AGG(last_fetched_at) AS last_fetched_timestamps, + ARRAY_AGG(scheduled_at) AS scheduled_timestamps + FROM + omnivore.subscriptions + WHERE + type = $1 + AND status = $2 + AND (scheduled_at <= NOW() OR scheduled_at IS NULL) + GROUP BY + url + `, + [SubscriptionType.Rss, SubscriptionStatus.Active] + )) as RssSubscriptionGroup[] + + logger.info('scheduledSubscriptions', subscriptionGroups) // create a cloud taks to fetch rss feed item for each subscription await Promise.all( - subscriptions.map((subscription) => { + subscriptionGroups.map((subscriptionGroup) => { try { - return enqueueRssFeedFetch(subscription.user.id, subscription) + return enqueueRssFeedFetch(subscriptionGroup) } catch (error) { logger.info('error creating rss feed fetch task', error) } diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 3a30469a8..9e6e78b37 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -1,5 +1,5 @@ import axios from 'axios' -import { DeleteResult } from 'typeorm' +import { DeepPartial, DeleteResult } from 'typeorm' import { appDataSource } from '../data_source' import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' @@ -215,3 +215,9 @@ export const deleteSubscription = async ( userId ) } + +export const createRssSubscriptions = async ( + subscriptions: DeepPartial[] +) => { + return getRepository(Subscription).save(subscriptions) +} diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index f7cc4c829..7e09968ea 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -7,7 +7,6 @@ import axios from 'axios' import { nanoid } from 'nanoid' import { DeepPartial } from 'typeorm' import { Recommendation } from '../entity/recommendation' -import { Subscription } from '../entity/subscription' import { env } from '../env' import { ArticleSavingRequestStatus, @@ -592,21 +591,29 @@ export const enqueueThumbnailTask = async ( return createdTasks[0].name } -export const enqueueRssFeedFetch = async ( - userId: string, - rssFeedSubscription: Subscription -): Promise => { - const { GOOGLE_CLOUD_PROJECT } = process.env - const payload = { - subscriptionId: rssFeedSubscription.id, - feedUrl: rssFeedSubscription.url, - lastFetchedAt: rssFeedSubscription.lastFetchedAt?.getTime() || 0, // unix timestamp in milliseconds - lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, - scheduledAt: new Date().getTime(), // unix timestamp in milliseconds - } +export interface RssSubscriptionGroup { + url: string + subscription_ids: string[] + user_ids: string[] + last_fetched_timestamps: (Date | null)[] + scheduled_timestamps: (Date | null)[] +} - const headers = { - [OmnivoreAuthorizationHeader]: generateVerificationToken({ id: userId }), +export const enqueueRssFeedFetch = async ( + subscriptionGroup: RssSubscriptionGroup +): Promise => { + const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env + const payload = { + subscriptionIds: subscriptionGroup.subscription_ids, + feedUrl: subscriptionGroup.url, + lastFetchedTimestamps: subscriptionGroup.last_fetched_timestamps.map( + (timestamp) => timestamp?.getTime() || 0 + ), // unix timestamp in milliseconds + lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, + scheduledTimestamps: subscriptionGroup.scheduled_timestamps.map( + (timestamp) => timestamp?.getTime() || 0 + ), // unix timestamp in milliseconds + userIds: subscriptionGroup.user_ids, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment @@ -615,9 +622,10 @@ export const enqueueRssFeedFetch = async ( // Calling the handler function directly. setTimeout(() => { axios - .post(env.queue.rssFeedTaskHandlerUrl, payload, { - headers, - }) + .post( + `${env.queue.rssFeedTaskHandlerUrl}?token=${PUBSUB_VERIFICATION_TOKEN}`, + payload + ) .catch((error) => { logError(error) }) @@ -630,8 +638,7 @@ export const enqueueRssFeedFetch = async ( project: GOOGLE_CLOUD_PROJECT, queue: 'omnivore-rss-queue', payload, - taskHandlerUrl: env.queue.rssFeedTaskHandlerUrl, - requestHeaders: headers, + taskHandlerUrl: `${env.queue.rssFeedTaskHandlerUrl}?token=${PUBSUB_VERIFICATION_TOKEN}`, }) if (!createdTasks || !createdTasks[0].name) { diff --git a/packages/api/test/routers/rss_feed.test.ts b/packages/api/test/routers/rss_feed.test.ts new file mode 100644 index 000000000..a8e35a345 --- /dev/null +++ b/packages/api/test/routers/rss_feed.test.ts @@ -0,0 +1,101 @@ +import chai, { expect } from 'chai' +import 'mocha' +import sinon from 'sinon' +import sinonChai from 'sinon-chai' +import { User } from '../../src/entity/user' +import { SubscriptionType } from '../../src/generated/graphql' +import { createRssSubscriptions } from '../../src/services/subscriptions' +import { deleteUser } from '../../src/services/user' +import * as createTask from '../../src/utils/createTask' +import { createTestUser } from '../db' +import { request } from '../util' + +chai.use(sinonChai) + +describe('Rss feeds Router', () => { + const token = process.env.PUBSUB_VERIFICATION_TOKEN || '' + + let user: User + let user1: User + let user2: User + + before(async () => { + // create test user and login + user = await createTestUser('fakeUser') + await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + + user1 = await createTestUser('fakeUser1') + user2 = await createTestUser('fakeUser2') + // create test subscriptions + const name1 = 'NPR' + const url1 = 'https://www.npr.org/rss/rss.php?id=1001' + const name2 = 'BBC' + const url2 = 'http://feeds.bbci.co.uk/news/rss.xml' + await createRssSubscriptions([ + { + name: name1, + user: { id: user1.id }, + scheduledAt: new Date(), + url: url1, + type: SubscriptionType.Rss, + }, + { + name: name1, + user: { id: user2.id }, + scheduledAt: new Date(), + url: url1, + type: SubscriptionType.Rss, + }, + { + name: name2, + user: { id: user1.id }, + url: url2, + type: SubscriptionType.Rss, + }, + { + name: name2, + user: { id: user2.id }, + // 1 hour in the future + scheduledAt: new Date(Date.now() + 60 * 60 * 1000), + url: url2, + type: SubscriptionType.Rss, + }, + ]) + }) + + after(async () => { + // clean up + await deleteUser(user.id) + await deleteUser(user1.id) + await deleteUser(user2.id) + }) + + it('fetches all scheduled RSS feeds', async () => { + const data = { + message: { + data: Buffer.from('').toString('base64'), + publishTime: new Date().toISOString(), + }, + } + + // fake enqueueRssFeedFetch function + const fake = sinon.replace( + createTask, + 'enqueueRssFeedFetch', + sinon.fake.resolves('task name') + ) + + const res = await request + .post('/svc/pubsub/rss-feed/fetchAll?token=' + token) + .send(data) + .expect(200) + expect(res.text).to.eql('OK') + + // check if enqueueRssFeedFetch is called + expect(fake).to.have.been.called + + sinon.restore() + }) +}) From 1880f2ace4e3a7977bc59a97f9c9c90265f64e60 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 17:25:07 +0800 Subject: [PATCH 061/115] Update rss handler to fetch a group of rss feeds in one task --- packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + .../api/src/resolvers/subscriptions/index.ts | 12 +- packages/api/src/routers/svc/rss_feed.ts | 9 +- packages/api/src/schema.ts | 1 + packages/api/src/utils/createTask.ts | 21 +- packages/rss-handler/src/index.ts | 341 ++++++++++-------- 7 files changed, 225 insertions(+), 161 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 0ad412b4f..6d04536da 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2980,6 +2980,7 @@ export type UpdateSubscriptionInput = { lastFetchedAt?: InputMaybe; lastFetchedChecksum?: InputMaybe; name?: InputMaybe; + scheduledAt?: InputMaybe; status?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index af9064bee..f50844aee 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2393,6 +2393,7 @@ input UpdateSubscriptionInput { lastFetchedAt: Date lastFetchedChecksum: String name: String + scheduledAt: Date status: SubscriptionStatus } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index 7bf145452..dd0b8a7c7 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -239,11 +239,12 @@ export const subscribeResolver = authorized< // create a cloud task to fetch rss feed item for the new subscription await enqueueRssFeedFetch({ - user_ids: [uid], + userIds: [uid], url: input.url, - subscription_ids: [newSubscription.id], - scheduled_timestamps: [null], - last_fetched_timestamps: [null], + subscriptionIds: [newSubscription.id], + scheduledDates: [new Date()], // fetch immediately + fetchedDates: [null], + checksums: [null], }) return { @@ -300,6 +301,9 @@ export const updateSubscriptionResolver = authorized< : undefined, lastFetchedChecksum: input.lastFetchedChecksum || undefined, status: input.status || undefined, + scheduledAt: input.scheduledAt + ? new Date(input.scheduledAt) + : undefined, }) return repo.findOneByOrFail({ diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 93077f6e4..053eb7d6f 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -30,10 +30,11 @@ export function rssFeedRouter() { ` SELECT url, - ARRAY_AGG(id) AS subscription_ids, - ARRAY_AGG(user_id) AS user_ids, - ARRAY_AGG(last_fetched_at) AS last_fetched_timestamps, - ARRAY_AGG(scheduled_at) AS scheduled_timestamps + ARRAY_AGG(id) AS "subscriptionIds", + ARRAY_AGG(user_id) AS "userIds", + ARRAY_AGG(last_fetched_at) AS "fetchedDates", + ARRAY_AGG(IFNULL(scheduled_at, NOW())) AS "scheduledDates", + ARRAY_AGG(last_fetched_checksum) AS "checksums" FROM omnivore.subscriptions WHERE diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 2cc4e8b58..4d125b10a 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2550,6 +2550,7 @@ const schema = gql` lastFetchedAt: Date lastFetchedChecksum: String status: SubscriptionStatus + scheduledAt: Date } union UpdateSubscriptionResult = diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 7e09968ea..f6f2ac6dc 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -593,10 +593,11 @@ export const enqueueThumbnailTask = async ( export interface RssSubscriptionGroup { url: string - subscription_ids: string[] - user_ids: string[] - last_fetched_timestamps: (Date | null)[] - scheduled_timestamps: (Date | null)[] + subscriptionIds: string[] + userIds: string[] + fetchedDates: (Date | null)[] + scheduledDates: Date[] + checksums: (string | null)[] } export const enqueueRssFeedFetch = async ( @@ -604,16 +605,16 @@ export const enqueueRssFeedFetch = async ( ): Promise => { const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env const payload = { - subscriptionIds: subscriptionGroup.subscription_ids, + subscriptionIds: subscriptionGroup.subscriptionIds, feedUrl: subscriptionGroup.url, - lastFetchedTimestamps: subscriptionGroup.last_fetched_timestamps.map( + lastFetchedTimestamps: subscriptionGroup.fetchedDates.map( (timestamp) => timestamp?.getTime() || 0 ), // unix timestamp in milliseconds - lastFetchedChecksum: rssFeedSubscription.lastFetchedChecksum || null, - scheduledTimestamps: subscriptionGroup.scheduled_timestamps.map( - (timestamp) => timestamp?.getTime() || 0 + lastFetchedChecksums: subscriptionGroup.checksums, + scheduledTimestamps: subscriptionGroup.scheduledDates.map((timestamp) => + timestamp.getTime() ), // unix timestamp in milliseconds - userIds: subscriptionGroup.user_ids, + userIds: subscriptionGroup.userIds, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 8b3021659..2c874f3c1 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -8,10 +8,12 @@ import { promisify } from 'util' import { CONTENT_FETCH_URL, createCloudTask } from './task' interface RssFeedRequest { - subscriptionId: string + subscriptionIds: string[] feedUrl: string - lastFetchedAt: number // unix timestamp in milliseconds - lastFetchedChecksum: string | undefined + lastFetchedTimestamps: number[] // unix timestamp in milliseconds + scheduledTimestamps: number[] // unix timestamp in milliseconds + lastFetchedChecksums: string[] + userIds: string[] } // link can be a string or an object @@ -19,7 +21,12 @@ type RssFeedItemLink = string | { $: { rel?: string; href: string } } function isRssFeedRequest(body: any): body is RssFeedRequest { return ( - 'subscriptionId' in body && 'feedUrl' in body && 'lastFetchedAt' in body + 'subscriptionIds' in body && + 'feedUrl' in body && + 'lastFetchedTimestamps' in body && + 'scheduledTimestamps' in body && + 'userIds' in body && + 'lastFetchedChecksums' in body ) } @@ -53,7 +60,8 @@ const sendUpdateSubscriptionMutation = async ( userId: string, subscriptionId: string, lastFetchedAt: Date, - lastFetchedChecksum: string + lastFetchedChecksum: string, + scheduledAt: Date ) => { const JWT_SECRET = process.env.JWT_SECRET const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT @@ -81,6 +89,7 @@ const sendUpdateSubscriptionMutation = async ( id: subscriptionId, lastFetchedAt, lastFetchedChecksum, + scheduledAt, }, }, }) @@ -155,10 +164,36 @@ const parser = new Parser({ 'updated', 'created', ], - feed: ['dc:date', 'lastBuildDate', 'pubDate'], + feed: [ + 'dc:date', + 'lastBuildDate', + 'pubDate', + 'syn:updatePeriod', + 'syn:updateFrequency', + 'sy:updatePeriod', + 'sy:updateFrequency', + ], }, }) +const getUpdatePeriodInHours = (updatePeriod: string) => { + switch (updatePeriod) { + case 'hourly': + return 1 + case 'daily': + return 24 + case 'weekly': + return 7 * 24 + case 'monthly': + return 30 * 24 + case 'yearly': + return 365 * 24 + default: + // default to hourly + return 1 + } +} + // get link following the order of preference: via, alternate, self const getLink = (links: RssFeedItemLink[]) => { // sort links by preference @@ -185,162 +220,182 @@ const getLink = (links: RssFeedItemLink[]) => { return sortedLinks.find((link) => !!link) } -export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( - async (req, res) => { - if (!process.env.JWT_SECRET) { - console.error('Missing JWT_SECRET in environment') - return res.status(500).send('INTERNAL_SERVER_ERROR') +const processSubscription = async ( + feedUrl: string, + userId: string, + subscriptionId: string, + fetchResult: { content: string; checksum: string }, + lastFetchedAt: number, + scheduledAt: number, + lastFetchedChecksum: string +) => { + let lastItemFetchedAt: Date | null = null + let lastValidItem: Item | null = null + + if (fetchResult.checksum === lastFetchedChecksum) { + console.log('feed has not been updated', feedUrl, lastFetchedChecksum) + return + } + const updatedLastFetchedChecksum = fetchResult.checksum + + // fetch feed + let itemCount = 0 + const feed = await parser.parseString(fetchResult.content) + console.log('Fetched feed', feed.title, new Date()) + + const feedPubDate = (feed['dc:date'] || + feed.pubDate || + feed.lastBuildDate) as string | undefined + console.log('Feed pub date', feedPubDate) + if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { + console.log('Skipping old feed', feedPubDate) + return + } + + // save each item in the feed + for (const item of feed.items) { + // use published or updated if isoDate is not available for atom feeds + item.isoDate = + item.isoDate || + (item.published as string) || + (item.updated as string) || + (item.created as string) + console.log('Processing feed item', item.links, item.isoDate) + + if (!item.links || item.links.length === 0) { + console.log('Invalid feed item', item) + continue } - const token = req.header('Omnivore-Authorization') - if (!token) { - console.error('Missing authorization header') - return res.status(401).send('UNAUTHORIZED') + item.link = getLink(item.links as RssFeedItemLink[]) + if (!item.link) { + console.log('Invalid feed item links', item.links) + continue + } + + console.log('Fetching feed item', item.link) + + const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() + // remember the last valid item + if ( + !lastValidItem || + (lastValidItem.isoDate && publishedAt > new Date(lastValidItem.isoDate)) + ) { + lastValidItem = item + } + + // Max limit per-feed update + if (itemCount > 99) { + continue + } + + // skip old items and items that were published before 24h + if ( + publishedAt < new Date(lastFetchedAt) || + publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) + ) { + console.log('Skipping old feed item', item.link) + continue + } + + const created = await createSavingItemTask(userId, feedUrl, item) + if (!created) { + console.error('Failed to create task for feed item', item.link) + continue + } + + // remember the last item fetched at + if (!lastItemFetchedAt || publishedAt > lastItemFetchedAt) { + lastItemFetchedAt = publishedAt + } + + itemCount = itemCount + 1 + } + + // no items saved + if (!lastItemFetchedAt) { + // the feed has been fetched before, no new valid items found + if (lastFetchedAt || !lastValidItem) { + console.log('No new valid items found') + return + } + + // the feed has never been fetched, save at least the last valid item + const created = await createSavingItemTask(userId, feedUrl, lastValidItem) + if (!created) { + console.error('Failed to create task for feed item', lastValidItem.link) + throw new Error('Failed to create task for feed item') + } + + lastItemFetchedAt = lastValidItem.isoDate + ? new Date(lastValidItem.isoDate) + : new Date() + } + + const updateFrequency = (feed['syn:updateFrequency'] || + feed['sy:updateFrequency'] || + 1) as number + const updatePeriod = (feed['syn:updatePeriod'] || + feed['sy:updatePeriod'] || + 'hourly') as string + const nextScheduledAt = + scheduledAt + + getUpdatePeriodInHours(updatePeriod) * 60 * 60 * 1000 * updateFrequency + + // update subscription lastFetchedAt + const updatedSubscription = await sendUpdateSubscriptionMutation( + userId, + subscriptionId, + lastItemFetchedAt, + updatedLastFetchedChecksum, + new Date(nextScheduledAt) + ) + console.log('Updated subscription', updatedSubscription) +} + +export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { + console.log('query does not include valid token') + return res.sendStatus(403) } try { - let userId: string - - try { - const decoded = jwt.verify(token, process.env.JWT_SECRET) as { - uid: string - } - userId = decoded.uid - } catch (e) { - console.error('Authorization error', e) - return res.status(401).send('UNAUTHORIZED') - } - if (!isRssFeedRequest(req.body)) { console.error('Invalid request body', req.body) return res.status(400).send('INVALID_REQUEST_BODY') } - const { feedUrl, subscriptionId, lastFetchedAt, lastFetchedChecksum } = - req.body - console.log('Processing feed', feedUrl, lastFetchedAt) - - let lastItemFetchedAt: Date | null = null - let lastValidItem: Item | null = null + const { + feedUrl, + subscriptionIds, + lastFetchedTimestamps, + scheduledTimestamps, + userIds, + lastFetchedChecksums, + } = req.body + console.log('Processing feed', feedUrl) const fetchResult = await fetchAndChecksum(feedUrl) - if (fetchResult.checksum === lastFetchedChecksum) { - console.log('feed has not been updated', feedUrl, lastFetchedChecksum) - return res.send('ok') - } - const updatedLastFetchedChecksum = fetchResult.checksum - // fetch feed - let itemCount = 0 - const feed = await parser.parseString(fetchResult.content) - console.log('Fetched feed', feed.title, new Date()) + for (let i = 0; i < subscriptionIds.length; i++) { + const subscriptionId = subscriptionIds[i] + const lastFetchedAt = lastFetchedTimestamps[i] + const scheduledAt = scheduledTimestamps[i] + const userId = userIds[i] + const lastFetchedChecksum = lastFetchedChecksums[i] - const feedPubDate = (feed['dc:date'] || - feed.pubDate || - feed.lastBuildDate) as string | undefined - console.log('Feed pub date', feedPubDate) - if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { - console.log('Skipping old feed', feedPubDate) - return res.send('ok') - } - - // save each item in the feed - for (const item of feed.items) { - // use published or updated if isoDate is not available for atom feeds - item.isoDate = - item.isoDate || - (item.published as string) || - (item.updated as string) || - (item.created as string) - console.log('Processing feed item', item.links, item.isoDate) - - if (!item.links || item.links.length === 0) { - console.log('Invalid feed item', item) - continue - } - - item.link = getLink(item.links as RssFeedItemLink[]) - if (!item.link) { - console.log('Invalid feed item links', item.links) - continue - } - - console.log('Fetching feed item', item.link) - - const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() - // remember the last valid item - if ( - !lastValidItem || - (lastValidItem.isoDate && - publishedAt > new Date(lastValidItem.isoDate)) - ) { - lastValidItem = item - } - - // Max limit per-feed update - if (itemCount > 99) { - continue - } - - // skip old items and items that were published before 24h - if ( - publishedAt < new Date(lastFetchedAt) || - publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) - ) { - console.log('Skipping old feed item', item.link) - continue - } - - const created = await createSavingItemTask(userId, feedUrl, item) - if (!created) { - console.error('Failed to create task for feed item', item.link) - continue - } - - // remember the last item fetched at - if (!lastItemFetchedAt || publishedAt > lastItemFetchedAt) { - lastItemFetchedAt = publishedAt - } - - itemCount = itemCount + 1 - } - - // no items saved - if (!lastItemFetchedAt) { - // the feed has been fetched before, no new valid items found - if (lastFetchedAt || !lastValidItem) { - console.log('No new valid items found') - return res.send('ok') - } - - // the feed has never been fetched, save at least the last valid item - const created = await createSavingItemTask( + await processSubscription( + subscriptionId, userId, feedUrl, - lastValidItem + fetchResult, + lastFetchedAt, + scheduledAt, + lastFetchedChecksum ) - if (!created) { - console.error( - 'Failed to create task for feed item', - lastValidItem.link - ) - return res.status(500).send('INTERNAL_SERVER_ERROR') - } - - lastItemFetchedAt = lastValidItem.isoDate - ? new Date(lastValidItem.isoDate) - : new Date() } - // update subscription lastFetchedAt - const updatedSubscription = await sendUpdateSubscriptionMutation( - userId, - subscriptionId, - lastItemFetchedAt, - updatedLastFetchedChecksum - ) - console.log('Updated subscription', updatedSubscription) - res.send('ok') } catch (e) { console.error('Error while parsing RSS feed', e) From 750ccb0c42061aeee89793f90bab93afab1cb43b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 17:44:53 +0800 Subject: [PATCH 062/115] fix conflicts with main --- packages/api/src/routers/svc/rss_feed.ts | 6 ++---- ...ion.sql => 0139.do.add_scheduled_at_to_subscription.sql} | 0 ...n.sql => 0139.undo.add_scheduled_at_to_subscription.sql} | 0 3 files changed, 2 insertions(+), 4 deletions(-) rename packages/db/migrations/{0138.do.add_scheduled_at_to_subscription.sql => 0139.do.add_scheduled_at_to_subscription.sql} (100%) rename packages/db/migrations/{0138.undo.add_scheduled_at_to_subscription.sql => 0139.undo.add_scheduled_at_to_subscription.sql} (100%) diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 053eb7d6f..3b6a21247 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -33,8 +33,8 @@ export function rssFeedRouter() { ARRAY_AGG(id) AS "subscriptionIds", ARRAY_AGG(user_id) AS "userIds", ARRAY_AGG(last_fetched_at) AS "fetchedDates", - ARRAY_AGG(IFNULL(scheduled_at, NOW())) AS "scheduledDates", - ARRAY_AGG(last_fetched_checksum) AS "checksums" + ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", + ARRAY_AGG(last_fetched_checksum) AS checksums FROM omnivore.subscriptions WHERE @@ -47,8 +47,6 @@ export function rssFeedRouter() { [SubscriptionType.Rss, SubscriptionStatus.Active] )) as RssSubscriptionGroup[] - logger.info('scheduledSubscriptions', subscriptionGroups) - // create a cloud taks to fetch rss feed item for each subscription await Promise.all( subscriptionGroups.map((subscriptionGroup) => { diff --git a/packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0139.do.add_scheduled_at_to_subscription.sql similarity index 100% rename from packages/db/migrations/0138.do.add_scheduled_at_to_subscription.sql rename to packages/db/migrations/0139.do.add_scheduled_at_to_subscription.sql diff --git a/packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0139.undo.add_scheduled_at_to_subscription.sql similarity index 100% rename from packages/db/migrations/0138.undo.add_scheduled_at_to_subscription.sql rename to packages/db/migrations/0139.undo.add_scheduled_at_to_subscription.sql From 25c0ee3b340656fcb4bbb2520c62b43d42fc11f7 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 21:08:54 +0800 Subject: [PATCH 063/115] return 1 for updateFrequency if it is not a number --- packages/rss-handler/src/index.ts | 34 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 2c874f3c1..28a87bd3d 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -176,7 +176,27 @@ const parser = new Parser({ }, }) -const getUpdatePeriodInHours = (updatePeriod: string) => { +const getUpdateFrequency = (feed: any) => { + const updateFrequency = (feed['syn:updateFrequency'] || + feed['sy:updateFrequency']) as string | undefined + + if (!updateFrequency) { + return 1 + } + + const frequency = parseInt(updateFrequency, 10) + if (isNaN(frequency)) { + return 1 + } + + return frequency +} + +const getUpdatePeriodInHours = (feed: any) => { + const updatePeriod = (feed['syn:updatePeriod'] || feed['sy:updatePeriod']) as + | string + | undefined + switch (updatePeriod) { case 'hourly': return 1 @@ -332,15 +352,9 @@ const processSubscription = async ( : new Date() } - const updateFrequency = (feed['syn:updateFrequency'] || - feed['sy:updateFrequency'] || - 1) as number - const updatePeriod = (feed['syn:updatePeriod'] || - feed['sy:updatePeriod'] || - 'hourly') as string - const nextScheduledAt = - scheduledAt + - getUpdatePeriodInHours(updatePeriod) * 60 * 60 * 1000 * updateFrequency + const updateFrequency = getUpdateFrequency(feed) + const updatePeriodInMs = getUpdatePeriodInHours(feed) * 60 * 60 * 1000 + const nextScheduledAt = scheduledAt + updatePeriodInMs * updateFrequency // update subscription lastFetchedAt const updatedSubscription = await sendUpdateSubscriptionMutation( From f750648824651a2088402985139762766c73d31a Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 21:46:43 +0800 Subject: [PATCH 064/115] fix importer triggers thumbnailer unexpectedly --- packages/puppeteer-parse/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b3413a244..b877b8eec 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -272,7 +272,7 @@ const sendSavePageMutation = async (userId, input) => { } }`, variables: { - input: Object.assign({}, input , { source: 'puppeteer-parse' }), + input: Object.assign({}, input), }, }); @@ -341,7 +341,7 @@ async function fetchContent(req, res) { const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); const state = req.body.state const labels = req.body.labels - const source = req.body.source || 'parseContent'; + const source = req.body.source || 'puppeteer-parse'; const taskId = req.body.taskId; // taskId is used to update import status const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); const locale = (req.query ? req.query.locale : undefined) || (req.body ? req.body.locale : undefined); @@ -473,6 +473,7 @@ async function fetchContent(req, res) { rssFeedUrl, savedAt, publishedAt, + source, }); if (!apiResponse) { logRecord.error = 'error while saving page'; From d746510358ec2cbbca0855f4d3bbabaf63f4f444 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 19 Oct 2023 21:50:16 +0800 Subject: [PATCH 065/115] cont --- packages/puppeteer-parse/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b877b8eec..78cdc87dd 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -272,7 +272,7 @@ const sendSavePageMutation = async (userId, input) => { } }`, variables: { - input: Object.assign({}, input), + input, }, }); From efb58f860400725970a7e2ec04ece04f1ae8e1f3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 10:06:28 +0800 Subject: [PATCH 066/115] avoid using string interop in graphql --- .../api/test/resolvers/subscriptions.test.ts | 24 +++++++------------ packages/api/test/util.ts | 9 ++++--- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 22ee107b1..085845eb4 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -350,17 +350,9 @@ describe('Subscriptions API', () => { }) describe('Subscribe API', () => { - const query = ( - name: string | null, - url: string | null, - subscriptionType: SubscriptionType | null - ) => ` - mutation { - subscribe(input: { - name: ${name ? `"${name}"` : null} - url: ${url ? `"${url}"` : null} - subscriptionType: ${subscriptionType} - }) { + const query = ` + mutation Subscribe($input: SubscribeInput!){ + subscribe(input: $input) { ... on SubscribeSuccess { subscriptions { id @@ -410,8 +402,9 @@ describe('Subscriptions API', () => { it('returns an error', async () => { const res = await graphqlRequest( - query(null, url, subscriptionType), - authToken + query, + authToken, + { input: { url, subscriptionType } }, ).expect(200) expect(res.body.data.subscribe.errorCodes).to.eql([ 'ALREADY_SUBSCRIBED', @@ -421,8 +414,9 @@ describe('Subscriptions API', () => { it('creates a rss subscription', async () => { const res = await graphqlRequest( - query(null, url, subscriptionType), - authToken + query, + authToken, + { input: { url, subscriptionType } }, ).expect(200) expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') diff --git a/packages/api/test/util.ts b/packages/api/test/util.ts index 02bac0f99..8268c1503 100644 --- a/packages/api/test/util.ts +++ b/packages/api/test/util.ts @@ -17,15 +17,14 @@ export const stopApolloServer = async () => { export const graphqlRequest = ( query: string, - authToken?: string + authToken: string, + variables?: Record, ): supertest.Test => { return request .post(apollo.graphqlPath) - .send({ - query, - }) + .send({ query, variables }) .set('Accept', 'application/json') - .set('authorization', authToken || '') + .set('authorization', authToken) .expect('Content-Type', /json/) } From 4e36a5809ea75a0a68aa0f1eb2c93177c2082c00 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 11:05:58 +0800 Subject: [PATCH 067/115] fix failure of re-subscribing rss feed --- packages/api/src/generated/graphql.ts | 3 +- packages/api/src/generated/schema.graphql | 3 +- .../api/src/resolvers/subscriptions/index.ts | 116 ++++++++++-------- packages/api/src/schema.ts | 3 +- packages/api/src/services/subscriptions.ts | 11 +- .../api/test/resolvers/subscriptions.test.ts | 43 +++++-- 6 files changed, 100 insertions(+), 79 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 6d04536da..3bbc9a88d 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2616,9 +2616,8 @@ export enum SubscribeErrorCode { } export type SubscribeInput = { - name?: InputMaybe; subscriptionType?: InputMaybe; - url?: InputMaybe; + url: Scalars['String']; }; export type SubscribeResult = SubscribeError | SubscribeSuccess; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index f50844aee..e5192b308 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -2058,9 +2058,8 @@ enum SubscribeErrorCode { } input SubscribeInput { - name: String subscriptionType: SubscriptionType - url: String + url: String! } union SubscribeResult = SubscribeError | SubscribeSuccess diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index dd0b8a7c7..8d5b74a83 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -175,25 +175,8 @@ export const subscribeResolver = authorized< SubscribeSuccessPartial, SubscribeError, MutationSubscribeArgs ->(async (_, { input }, { authTrx, uid, log }) => { - log.info('subscribeResolver') - +>(async (_, { input }, { uid, log }) => { try { - // find existing subscription - const subscription = await authTrx((t) => - t.getRepository(Subscription).findOneBy({ - url: input.url || undefined, - name: input.name || undefined, - user: { id: uid }, - type: input.subscriptionType || SubscriptionType.Rss, // default to rss - }) - ) - if (subscription) { - return { - errorCodes: [SubscribeErrorCode.AlreadySubscribed], - } - } - analytics.track({ userId: uid, event: 'subscribed', @@ -203,58 +186,83 @@ export const subscribeResolver = authorized< }, }) - // create new rss subscription - if (input.url) { - const MAX_RSS_SUBSCRIPTIONS = 150 - // validate rss feed - const feed = await parser.parseURL(input.url) - - // limit number of rss subscriptions to 50 - const newSubscriptions = (await authTrx((t) => - t.query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) - select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions - where user_id = $5 and type = 'RSS' and status = 'ACTIVE' - having count(*) < $7 - returning *;`, - [ - feed.title, - input.url, - feed.description || null, - SubscriptionType.Rss, - uid, - feed.image?.url || null, - MAX_RSS_SUBSCRIPTIONS, - ] - ) - )) as Subscription[] - - if (newSubscriptions.length === 0) { + // find existing subscription + const existingSubscription = await getRepository(Subscription).findOneBy({ + url: input.url, + user: { id: uid }, + type: SubscriptionType.Rss, + }) + if (existingSubscription) { + if (existingSubscription.status === SubscriptionStatus.Active) { return { - errorCodes: [SubscribeErrorCode.ExceededMaxSubscriptions], + errorCodes: [SubscribeErrorCode.AlreadySubscribed], } } - const newSubscription = newSubscriptions[0] + // re-subscribe + const updatedSubscription = await getRepository(Subscription).save({ + ...existingSubscription, + status: SubscriptionStatus.Active, + }) - // create a cloud task to fetch rss feed item for the new subscription + // create a cloud task to fetch rss feed item for resub subscription await enqueueRssFeedFetch({ userIds: [uid], url: input.url, - subscriptionIds: [newSubscription.id], + subscriptionIds: [updatedSubscription.id], scheduledDates: [new Date()], // fetch immediately - fetchedDates: [null], - checksums: [null], + fetchedDates: [updatedSubscription.lastFetchedAt || null], + checksums: [updatedSubscription.lastFetchedChecksum || null], }) return { - subscriptions: [newSubscription], + subscriptions: [updatedSubscription], } } - log.info('missing url or name') + // create new rss subscription + const MAX_RSS_SUBSCRIPTIONS = 150 + // validate rss feed + const feed = await parser.parseURL(input.url) + + // limit number of rss subscriptions to 150 + const results = (await getRepository(Subscription).query( + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) + select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions + where user_id = $5 and type = 'RSS' and status = 'ACTIVE' + having count(*) < $7 + returning *;`, + [ + feed.title, + input.url, + feed.description || null, + SubscriptionType.Rss, + uid, + feed.image?.url || null, + MAX_RSS_SUBSCRIPTIONS, + ] + )) as Subscription[] + + if (results.length === 0) { + return { + errorCodes: [SubscribeErrorCode.ExceededMaxSubscriptions], + } + } + + const newSubscription = results[0] + + // create a cloud task to fetch rss feed item for the new subscription + await enqueueRssFeedFetch({ + userIds: [uid], + url: input.url, + subscriptionIds: [newSubscription.id], + scheduledDates: [new Date()], // fetch immediately + fetchedDates: [null], + checksums: [null], + }) + return { - errorCodes: [SubscribeErrorCode.BadRequest], + subscriptions: [newSubscription], } } catch (error) { log.error('failed to subscribe', error) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 4d125b10a..de3f5209c 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -2538,8 +2538,7 @@ const schema = gql` } input SubscribeInput { - url: String - name: String + url: String! subscriptionType: SubscriptionType } diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 9e6e78b37..fcf25af01 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -205,15 +205,8 @@ export const createSubscription = async ( }) } -export const deleteSubscription = async ( - id: string, - userId: string -): Promise => { - return authTrx( - (tx) => tx.getRepository(Subscription).delete(id), - undefined, - userId - ) +export const deleteSubscription = async (id: string): Promise => { + return getRepository(Subscription).delete(id) } export const createRssSubscriptions = async ( diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 085845eb4..02f7cd1b8 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -397,21 +397,47 @@ describe('Subscriptions API', () => { }) after(async () => { - await deleteSubscription(existingSubscription.id, user.id) + await deleteSubscription(existingSubscription.id) }) it('returns an error', async () => { - const res = await graphqlRequest( - query, - authToken, - { input: { url, subscriptionType } }, - ).expect(200) + const res = await graphqlRequest(query, authToken, { + input: { url, subscriptionType }, + }).expect(200) expect(res.body.data.subscribe.errorCodes).to.eql([ 'ALREADY_SUBSCRIBED', ]) }) }) + context('when the user unsubscribed the feed', () => { + let existingSubscription: Subscription + + before(async () => { + existingSubscription = await createSubscription( + user.id, + 'RSS Feed', + undefined, + SubscriptionStatus.Unsubscribed, + url, + subscriptionType, + url + ) + }) + + after(async () => { + await deleteSubscription(existingSubscription.id) + }) + + it('re-subscribes the user', async () => { + const res = await graphqlRequest(query, authToken, { + input: { url, subscriptionType }, + }).expect(200) + expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) + expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') + }) + }) + it('creates a rss subscription', async () => { const res = await graphqlRequest( query, @@ -422,10 +448,7 @@ describe('Subscriptions API', () => { expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') // clean up - await deleteSubscription( - res.body.data.subscribe.subscriptions[0].id, - user.id - ) + await deleteSubscription(res.body.data.subscribe.subscriptions[0].id) }) }) }) From b9e2f9ee53f0bf53aff2880f542bb01e51f52fe5 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 20 Oct 2023 11:00:31 +0800 Subject: [PATCH 068/115] Add a longer rate limit window on createaccount/reset password, reduce api rate limit hits --- packages/api/src/routers/auth/auth_router.ts | 12 ++++++++++++ packages/api/src/server.ts | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 333391d62..3b38048a3 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -46,6 +46,7 @@ import { } from './google_auth' import { createWebAuthToken } from './jwt_helpers' import { createMobileAccountCreationResponse } from './mobile/account_creation' +import rateLimit from 'express-rate-limit' export interface SignupRequest { email: string @@ -80,6 +81,14 @@ export const isValidSignupRequest = (obj: any): obj is SignupRequest => { ) } +// The hourly limiter is used on the create account, +// and reset password endpoints +// this limits users to five operations per an hour +const hourlyLimiter = rateLimit({ + windowMs: 60 * 60 * 1000, + max: 5, +}) + export function authRouter() { const router = express.Router() @@ -108,6 +117,7 @@ export function authRouter() { ) router.post( '/create-account', + hourlyLimiter, cors(corsConfig), async (req, res) => { const { name, bio, username } = req.body @@ -480,6 +490,7 @@ export function authRouter() { router.post( '/email-signup', + hourlyLimiter, cors(corsConfig), async (req: express.Request, res: express.Response) => { if (!isValidSignupRequest(req.body)) { @@ -599,6 +610,7 @@ export function authRouter() { router.post( '/forgot-password', + hourlyLimiter, cors(corsConfig), async (req: express.Request, res: express.Response) => { const email = req.body.email?.trim() as string // trim whitespace diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 9e41af64d..b40a82bb8 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -68,7 +68,7 @@ export const createApp = (): { const token = getTokenByRequest(req) try { const claims = await getClaimsByToken(token) - return claims ? 100 : 15 + return claims ? 60 : 15 } catch (e) { console.log('non-authenticated request') return 15 From a23b67b70b8e7315ca09b60f5d94f8886fe11378 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 20 Oct 2023 11:44:52 +0800 Subject: [PATCH 069/115] Dont rate limit auth in local env so tests can run --- packages/api/src/routers/auth/auth_router.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 3b38048a3..7779b9af8 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -87,6 +87,7 @@ export const isValidSignupRequest = (obj: any): obj is SignupRequest => { const hourlyLimiter = rateLimit({ windowMs: 60 * 60 * 1000, max: 5, + skip: (req) => env.dev.isLocal, }) export function authRouter() { From 23eb7ea76fdc83d569f57785254aeb5ec5cbc265 Mon Sep 17 00:00:00 2001 From: Rudra Date: Fri, 20 Oct 2023 11:51:02 +0530 Subject: [PATCH 070/115] fix typo --- packages/api/src/resolvers/reaction/index.ts | 2 +- packages/api/src/routers/svc/content.ts | 2 +- packages/pdf-handler/src/pdf.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/src/resolvers/reaction/index.ts b/packages/api/src/resolvers/reaction/index.ts index aec16962a..ceea66324 100644 --- a/packages/api/src/resolvers/reaction/index.ts +++ b/packages/api/src/resolvers/reaction/index.ts @@ -17,7 +17,7 @@ export type PartialCreateReactionSuccess = Merge< // if ((!userArticleId && !highlightId) || (userArticleId && highlightId)) { // // One reaction target is required -// // Higlight replies hasn't supported yet +// // Highlight replies hasn't supported yet // return { // errorCodes: [CreateReactionErrorCode.BadTarget], // } diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index ce184ea5b..70723429b 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -91,7 +91,7 @@ export function contentServiceRouter() { if (msg.description) itemToUpdate.description = msg.description // This event is fired after the file is fully uploaded, - // so along with updateing content, we mark it as + // so along with updating content, we mark it as // succeeded. itemToUpdate.state = LibraryItemState.Succeeded diff --git a/packages/pdf-handler/src/pdf.ts b/packages/pdf-handler/src/pdf.ts index 48d85b637..0585b742b 100644 --- a/packages/pdf-handler/src/pdf.ts +++ b/packages/pdf-handler/src/pdf.ts @@ -14,7 +14,7 @@ interface Page { lines: string[] } -// Unused at the moment -- comented out for now to satisfy linter +// Unused at the moment -- commented out for now to satisfy linter const MAX_TITLE_LENGTH = 95 type MetadataInfoKey = From 6618a6ed19af4cdf679976d6ba69445b9b83825b Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 14:31:16 +0800 Subject: [PATCH 071/115] migrate popular reads into database --- packages/api/src/repository/library_item.ts | 42 + .../api/src/resolvers/popular_reads/index.ts | 6 +- packages/api/src/services/popular_reads.ts | 192 +- .../popular_reads/elad_meetings-content.html | 36 - .../popular_reads/elad_meetings-original.html | 1497 --- .../jonbo_digital_tools-content.html | 208 - .../jonbo_digital_tools-original.html | 453 - .../omnivore_get_started-content.html | 331 - .../omnivore_get_started-original.html | 1670 --- .../popular_reads/omnivore_ios-content.html | 94 - .../popular_reads/omnivore_ios-original.html | 1456 --- .../omnivore_organize-content.html | 73 - .../omnivore_organize-original.html | 1022 -- .../power_read_it_later-content.html | 239 - .../power_read_it_later-original.html | 1551 --- .../popular_reads/rlove_carnitas-content.html | 34 - .../rlove_carnitas-original.html | 1230 -- .../db/migrations/0140.do.popular_read.sql | 9931 +++++++++++++++++ .../db/migrations/0140.undo.popular_read.sql | 9 + 19 files changed, 10007 insertions(+), 10067 deletions(-) delete mode 100644 packages/api/src/services/popular_reads/elad_meetings-content.html delete mode 100644 packages/api/src/services/popular_reads/elad_meetings-original.html delete mode 100644 packages/api/src/services/popular_reads/jonbo_digital_tools-content.html delete mode 100644 packages/api/src/services/popular_reads/jonbo_digital_tools-original.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_get_started-content.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_get_started-original.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_ios-content.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_ios-original.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_organize-content.html delete mode 100644 packages/api/src/services/popular_reads/omnivore_organize-original.html delete mode 100644 packages/api/src/services/popular_reads/power_read_it_later-content.html delete mode 100644 packages/api/src/services/popular_reads/power_read_it_later-original.html delete mode 100644 packages/api/src/services/popular_reads/rlove_carnitas-content.html delete mode 100644 packages/api/src/services/popular_reads/rlove_carnitas-original.html create mode 100755 packages/db/migrations/0140.do.popular_read.sql create mode 100755 packages/db/migrations/0140.undo.popular_read.sql diff --git a/packages/api/src/repository/library_item.ts b/packages/api/src/repository/library_item.ts index 1a428d65b..1e7e3d708 100644 --- a/packages/api/src/repository/library_item.ts +++ b/packages/api/src/repository/library_item.ts @@ -17,4 +17,46 @@ export const libraryItemRepository = appDataSource countByCreatedAt(createdAt: Date) { return this.countBy({ createdAt }) }, + + createByPopularRead(name: string, userId: string) { + return this.query( + ` + INSERT INTO omnivore.library_item ( + slug, + readable_content, + original_content, + description, + title, + author, + original_url, + item_type, + thumbnail, + published_at, + site_name, + user_id, + word_count + ) + SELECT + slug, + readable_content, + original_content, + description, + title, + author, + original_url, + $1, + thumbnail, + published_at, + site_name, + $2, + word_count + FROM + omnivore.popular_read + WHERE + key = $3 + RETURNING * + `, + ['ARTICLE', userId, name] + ) as Promise + }, }) diff --git a/packages/api/src/resolvers/popular_reads/index.ts b/packages/api/src/resolvers/popular_reads/index.ts index 3f91fe2c9..30e1de08c 100644 --- a/packages/api/src/resolvers/popular_reads/index.ts +++ b/packages/api/src/resolvers/popular_reads/index.ts @@ -11,12 +11,12 @@ export const addPopularReadResolver = authorized< AddPopularReadError, MutationAddPopularReadArgs >(async (_, { name }, { uid }) => { - const item = await addPopularRead(uid, name) - if (!item) { + const items = await addPopularRead(uid, name) + if (items.length === 0) { return { errorCodes: [AddPopularReadErrorCode.NotFound] } } return { - pageId: item.id, + pageId: items[0].id, } }) diff --git a/packages/api/src/services/popular_reads.ts b/packages/api/src/services/popular_reads.ts index c98fd9851..de6daab4d 100644 --- a/packages/api/src/services/popular_reads.ts +++ b/packages/api/src/services/popular_reads.ts @@ -1,93 +1,23 @@ import * as httpContext from 'express-http-context2' -import { readFileSync } from 'fs' -import path from 'path' -import { DeepPartial, EntityManager } from 'typeorm' +import { EntityManager } from 'typeorm' import { appDataSource } from '../data_source' -import { LibraryItem } from '../entity/library_item' -import { PageType } from '../generated/graphql' import { authTrx } from '../repository' import { libraryItemRepository } from '../repository/library_item' -import { generateSlug, stringToHash, wordsCount } from '../utils/helpers' import { logger } from '../utils/logger' -import { createLibraryItem } from './library_item' -type PopularRead = { - url: string - title: string - author: string - description: string - previewImage?: string - publishedAt: Date - siteName: string - - content: string - originalHtml: string -} - -const popularRead = (key: string): PopularRead | undefined => { - const metadata = popularReads.find((pr) => pr.key === key) - if (!metadata) { - return undefined - } - - try { - const content = readFileSync( - path.resolve(__dirname, `popular_reads/${key}-content.html`), - 'utf8' - ) - const originalHtml = readFileSync( - path.resolve(__dirname, `./popular_reads/${key}-original.html`), - 'utf8' - ) - if (!content || !originalHtml) { - return undefined - } - - return { - ...metadata, - content, - originalHtml, - } - } catch (e) { - logger.info('error adding popular read', e) - return undefined - } -} - -const popularReadToLibraryItem = ( +export const addPopularRead = async ( + userId: string, name: string, - userId: string -): DeepPartial | null => { - const pr = popularRead(name) - if (!pr) { - return null - } - - return { - slug: generateSlug(pr.title), - readableContent: pr.content, - originalContent: pr.originalHtml, - description: pr.description, - title: pr.title, - author: pr.author, - originalUrl: pr.url, - itemType: PageType.Article, - textContentHash: stringToHash(pr.content), - thumbnail: pr.previewImage, - publishedAt: pr.publishedAt, - siteName: pr.siteName, - user: { id: userId }, - wordCount: wordsCount(pr.content), - } -} - -export const addPopularRead = async (userId: string, name: string) => { - const itemToSave = popularReadToLibraryItem(name, userId) - if (!itemToSave) { - return null - } - - return createLibraryItem(itemToSave, userId) + entityManager?: EntityManager +) => { + return authTrx( + async (tx) => + tx + .withRepository(libraryItemRepository) + .createByPopularRead(name, userId), + entityManager, + userId + ) } const addPopularReads = async ( @@ -95,15 +25,15 @@ const addPopularReads = async ( userId: string, entityManager: EntityManager ) => { - const libraryItems = names - .map((name) => popularReadToLibraryItem(name, userId)) - .filter((pr) => pr !== null) as DeepPartial[] - - return authTrx( - async (tx) => tx.withRepository(libraryItemRepository).save(libraryItems), - entityManager, - userId - ) + // insert one by one to ensure that the order is preserved + for (const name of names) { + try { + await addPopularRead(userId, name, entityManager) + } catch (error) { + logger.error('failed to add popular read', error) + continue + } + } } export const addPopularReadsForNewUser = async ( @@ -132,81 +62,3 @@ export const addPopularReadsForNewUser = async ( defaultReads.push('omnivore_get_started') await addPopularReads(defaultReads, userId, em) } - -const popularReads = [ - { - key: 'omnivore_get_started', - url: 'https://blog.omnivore.app/p/getting-started-with-omnivore', - title: 'Getting Started with Omnivore', - author: 'The Omnivore Team', - description: 'Get the most out of Omnivore by learning how to use it.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sxQnqya1QNApB7ZAGPj9K20AU6sw0UAnjmAIy2ub8hUU/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F658efff4-341a-4720-8cf6-9b2bdbedfaa7_800x668.gif', - publishedAt: new Date('2021-10-13'), - siteName: 'Omnivore Blog', - }, - { - key: 'omnivore_ios', - url: 'https://blog.omnivore.app/p/saving-links-from-your-iphone-or', - title: 'Saving Links from Your iPhone or iPad', - author: 'Omnivore', - description: 'Learn how to save articles on iOS.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sWDfv7sARTIdAlx6Rw_6t-QwL3T9aniEJRa1-jVaglNg/https://substackcdn.com/image/youtube/w_728,c_limit/k6RkIqepAig', - publishedAt: new Date('2021-10-19'), - siteName: 'Omnivore Blog', - }, - { - key: 'omnivore_organize', - url: 'https://blog.omnivore.app/p/organize-your-omnivore-library-with', - title: 'Organize your Omnivore library with labels', - author: 'The Omnivore Team', - description: 'Use labels to organize your Omnivore library.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sTgJ5Q0XIg_EHdmPWcxtXFmkjn8T6hkJt7S9ziClagYo/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fdaf07af7-5cdb-4ecc-aace-1a46de3e9c58_1827x1090.png', - publishedAt: new Date('2022-04-18'), - siteName: 'Omnivore Blog', - }, - { - key: 'rlove_carnitas', - url: 'https://medium.com/@rlove/carnitas-ff0ef1044ae9', - title: 'Slow-Braised Carnitas Recipe', - author: 'Robert Love', - description: - 'Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. This is not an authentic recipe.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/88x88,sIcDXt3Ar0baKG1e1Yi1e2VUZFL85xPlOeEfAxF-s-Nw/https://miro.medium.com/max/1200/1*Wl-dMBJpSgPUxUOnPQthyg.jpeg', - publishedAt: new Date('2017-02-24'), - siteName: '@rlove', - }, - { - key: 'power_read_it_later', - url: 'https://fortelabs.co/blog/the-secret-power-of-read-it-later-apps', - title: 'The Secret Power of ‘Read It Later’ Apps', - author: 'Tiago Forte', - description: - 'At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sGN5R34M5z068QMXDZD32CQD6mCbxc47hWXm__JVUePE/https://fortelabs.com/wp-content/uploads/2015/11/1rPXwIczUJRCE54v8FfAHGw.jpeg', - publishedAt: new Date('2022-01-24'), - siteName: 'Forte Labs', - }, - { - key: 'elad_meetings', - url: 'http://blog.eladgil.com/2018/07/meeting-etiquette.html', - title: 'Better Meetings', - author: 'Elad Gil', - description: 'How to make meetings more productive.', - publishedAt: new Date('2018-07-02'), - siteName: 'Elad Blog', - }, - { - key: 'jonbo_digital_tools', - url: 'https://jon.bo/posts/digital-tools/', - title: 'Digital Tools I Wish Existed', - author: 'Jonathan Borichevskiy', - description: `My digital life in a nutshell: I discover relevant content I don’t have time to consume...`, - publishedAt: new Date('2019-11-28'), - siteName: 'JON.BO', - }, -] diff --git a/packages/api/src/services/popular_reads/elad_meetings-content.html b/packages/api/src/services/popular_reads/elad_meetings-content.html deleted file mode 100644 index 34f8a7aee..000000000 --- a/packages/api/src/services/popular_reads/elad_meetings-content.html +++ /dev/null @@ -1,36 +0,0 @@ -
-
-

As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
-
1. Determine who is necessary in the meeting.
- Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
-
2. Send out an agenda in advance.
- What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
-
Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
-
3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
- If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
-
If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
-
4. Kick off the meeting with objectives.
- Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
-
5. Assign a note taker.
- Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
-
6. Send out meeting notes.
- Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
-
Meeting notes would optimally include:
- a. Subject/topic of meeting
- b. Date
- c. Attendees
- d. Actions/decisions
- e. Agenda
- f. Detailed notes
-
7. Clean up the meeting calendar ongoing.

- Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
-
NOTES

- [1] There may be legal reasons to not take notes in specific meetings. Consult with the company's GC or external counsel on training for meeting note etiquette. -

-

MY BOOK -

-

You can pre-order the High Growth Handbook here.
-
-

-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/elad_meetings-original.html b/packages/api/src/services/popular_reads/elad_meetings-original.html deleted file mode 100644 index 054b6f5f9..000000000 --- a/packages/api/src/services/popular_reads/elad_meetings-original.html +++ /dev/null @@ -1,1497 +0,0 @@ - - - - - - - - - - - - - - - - - - Elad Blog: Better Meetings - - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- Monday, July 2, 2018 -

-
-
-
- - -

- Better Meetings -

-
-
-
-
-
- As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
-
1. Determine who is necessary in the meeting.
- Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
-
2. Send out an agenda in advance.
- What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
-
Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
-
3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
- If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
-
If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
-
4. Kick off the meeting with objectives.
- Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
-
5. Assign a note taker.
- Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
-
6. Send out meeting notes.
- Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
-
Meeting notes would optimally include:
- a. Subject/topic of meeting
- b. Date
- c. Attendees
- d. Actions/decisions
- e. Agenda
- f. Detailed notes
-
7. Clean up the meeting calendar ongoing.

- Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
-
NOTES

- [1] There may be legal reasons to not take notes in specific meetings. Consult with the company's GC or external counsel on training for meeting note etiquette.
-
-
-
-
- MY BOOK -
You can pre-order the High Growth Handbook here.
-

-
-
-
-
- RELATED POSTS -
- - - - - - -
-
-
- -
-
- -
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
- - - - - - diff --git a/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html b/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html deleted file mode 100644 index 711fe33ef..000000000 --- a/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html +++ /dev/null @@ -1,208 +0,0 @@ -
-
-

My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later.

-

There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed:

-
    -
  • -

    The more seamless the acquisition & ingestion, the more engaged I am with the content

    -
  • -
  • -

    Insights are just as likely to be found in a 400-page book as in a 40-minute podcast

    -
  • -
  • -

    Notes and their subsequent review are essential for long-term retention

    -
  • -
  • -

    Recommendations from other humans are as good, if not better, than algorithmic suggestions

    -
  • -
-

In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here.

-

Queue management for inbound digital content # -

-

Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance:

-
    -
  • -

    Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more.

    -
  • -
  • -

    Every book, article, post, or tweet has the potential to lead to more content.

    -
  • -
  • -

    Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs.

    -
  • -
  • -

    I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee?

    -
  • -
  • -

    Learning, work, news, and entertainment all have different priorities in my life (roughly in that order).

    -
  • -
  • -

    I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month.

    -
  • -
  • -

    I’m not always connected to a stable internet connection.

    -
  • -
  • -

    If it’s a long piece of content I want my position saved reliably so I can resume at a later point.

    -
  • -
  • -

    I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation.

    -
  • -
  • -

    I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place.

    -
  • -
  • -

    I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat.

    -
  • -
-
- -
Relevant XKCD, as is tradition
-
-

Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one.

-

Honorable Mentions: Pocket, Instapaper -

-

A universal book log, recommendation & sharing system # -

-

I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof.

-

Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry.

-

Whatever this tool for managing my ever-growing reading list will be, it should:

-
    -
  • -

    Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests.

    -
  • -
  • -

    Allow me to tag books instead of placing them into static lists (think clusters or tag clouds).

    -
  • -
  • -

    Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to.

    -
  • -
  • -

    Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it.

    -
  • -
  • -

    Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was.

    -
  • -
  • -

    Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on.

    -
  • -
  • -

    Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from.

    -
  • -
-

Honorable Mentions: None :(

-

Intelligent PDF viewers, eBook readers, audiobook & podcast players # -

-
- -
Functionality I want in my document reader
-
-

Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to:

-
    -
  • -

    Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them.

    -
  • -
  • -

    See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition.

    -
  • -
  • -

    View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know!

    -
  • -
  • -

    Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019.

    -
  • -
  • -

    Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues.

    -
  • -
  • -

    Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points).

    -
  • -
  • -

    Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading.

    -
  • -
-
- -
What I want my audiobook player to look like
-
-

Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could:

-
    -
  • -

    Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street.

    -
  • -
  • -

    View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps?

    -
  • -
-

Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar -

-

A centralized search interface for my digital brain (memex) # -

-

I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything.

-

I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should:

-
    -
  • -

    Accept and parse the following queries:

    -
      -
    • -

      spacex announcement type:video 2016

      -
    • -
    • -

      links from:jon@test.org topic:python

      -
    • -
    • -

      paper on temperature, productivity referenced in book:Uninhabitable Earth

      -
    • -
    • -

      type:pdf habits digital interfaces

      -
    • -
    • -

      reading comprehension type:blog post

      -
    • -
    • -

      printer ink receipt

      -
    • -
    • -

      type:book read:2017 finance

      -
    • -
    • -

      file:py datetime parse

      -
    • -
    -
  • -
  • -

    Respect my privacy: hosted on something I control and never mined for ads.

    -
  • -
  • -

    Support all my devices with two-way sync so I can search and add to it wherever I am.

    -
  • -
  • -

    Be extensible: allow me to easily ingest my own information and extend with desired functionality.

    -
  • -
  • -

    Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of.

    -
  • -
  • -

    Notify me about changes to documents and webpages I’ve visited.

    -
  • -
  • -

    Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it.

    -
  • -
-

Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop -

-

Parting Thoughts # -

-

I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools.

-

It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share.

-

I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback.

-

- HN Discussion -

-

Appendix # -

-

- The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS -

-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html b/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html deleted file mode 100644 index 66ff99044..000000000 --- a/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html +++ /dev/null @@ -1,453 +0,0 @@ - - - - - Digital Tools I Wish Existed :: up & to the right — Jonathan Borichevskiy - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - -
-
-
-

- Digital Tools I Wish Existed -

-

- -
-

- My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later. -

-

- There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed: -

-
    -
  • -

    - The more seamless the acquisition & ingestion, the more engaged I am with the content -

    -
  • -
  • -

    - Insights are just as likely to be found in a 400-page book as in a 40-minute podcast -

    -
  • -
  • -

    - Notes and their subsequent review are essential for long-term retention -

    -
  • -
  • -

    - Recommendations from other humans are as good, if not better, than algorithmic suggestions -

    -
  • -
-

- In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here. -

-

- Queue management for inbound digital content # -

-

- Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance: -

-
    -
  • -

    - Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more. -

    -
  • -
  • -

    - Every book, article, post, or tweet has the potential to lead to more content. -

    -
  • -
  • -

    - Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs. -

    -
  • -
  • -

    - I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee? -

    -
  • -
  • -

    - Learning, work, news, and entertainment all have different priorities in my life (roughly in that order). -

    -
  • -
  • -

    - I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month. -

    -
  • -
  • -

    - I’m not always connected to a stable internet connection. -

    -
  • -
  • -

    - If it’s a long piece of content I want my position saved reliably so I can resume at a later point. -

    -
  • -
  • -

    - I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation. -

    -
  • -
  • -

    - I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place. -

    -
  • -
  • -

    - I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat. -

    -
  • -
-
- -
- Relevant XKCD, as is tradition -
-
-

- Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one. -

-

- Honorable Mentions: Pocket, Instapaper -

-

- A universal book log, recommendation & sharing system # -

-

- I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof. -

-

- Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry. -

-

- Whatever this tool for managing my ever-growing reading list will be, it should: -

-
    -
  • -

    - Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests. -

    -
  • -
  • -

    - Allow me to tag books instead of placing them into static lists (think clusters or tag clouds). -

    -
  • -
  • -

    - Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to. -

    -
  • -
  • -

    - Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it. -

    -
  • -
  • -

    - Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was. -

    -
  • -
  • -

    - Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on. -

    -
  • -
  • -

    - Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from. -

    -
  • -
-

- Honorable Mentions: None :( -

-

- Intelligent PDF viewers, eBook readers, audiobook & podcast players # -

-
- -
- Functionality I want in my document reader -
-
-

- Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to: -

-
    -
  • -

    - Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them. -

    -
  • -
  • -

    - See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition. -

    -
  • -
  • -

    - View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know! -

    -
  • -
  • -

    - Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019. -

    -
  • -
  • -

    - Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues. -

    -
  • -
  • -

    - Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points). -

    -
  • -
  • -

    - Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading. -

    -
  • -
-
- -
- What I want my audiobook player to look like -
-
-

- Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could: -

-
    -
  • -

    - Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street. -

    -
  • -
  • -

    - View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps? -

    -
  • -
-

- Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar -

-

- A centralized search interface for my digital brain (memex) # -

-

- I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything. -

-

- I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should: -

-
    -
  • -

    - Accept and parse the following queries: -

    -
      -
    • -

      - spacex announcement type:video 2016 -

      -
    • -
    • -

      - links from:jon@test.org topic:python -

      -
    • -
    • -

      - paper on temperature, productivity referenced in book:Uninhabitable Earth -

      -
    • -
    • -

      - type:pdf habits digital interfaces -

      -
    • -
    • -

      - reading comprehension type:blog post -

      -
    • -
    • -

      - printer ink receipt -

      -
    • -
    • -

      - type:book read:2017 finance -

      -
    • -
    • -

      - file:py datetime parse -

      -
    • -
    -
  • -
  • -

    - Respect my privacy: hosted on something I control and never mined for ads. -

    -
  • -
  • -

    - Support all my devices with two-way sync so I can search and add to it wherever I am. -

    -
  • -
  • -

    - Be extensible: allow me to easily ingest my own information and extend with desired functionality. -

    -
  • -
  • -

    - Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of. -

    -
  • -
  • -

    - Notify me about changes to documents and webpages I’ve visited. -

    -
  • -
  • -

    - Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it. -

    -
  • -
-

- Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop -

-

- Parting Thoughts # -

-

- I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools. -

-

- It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share. -

-

- I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback. -

-

- HN Discussion -

-

- 2019-12-09: fixes grammar -

-

- Appendix # -

-

- The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS -

-
- -
-
- - - -
- - diff --git a/packages/api/src/services/popular_reads/omnivore_get_started-content.html b/packages/api/src/services/popular_reads/omnivore_get_started-content.html deleted file mode 100644 index d25355959..000000000 --- a/packages/api/src/services/popular_reads/omnivore_get_started-content.html +++ /dev/null @@ -1,331 +0,0 @@ -
-
-
-

Omnivore is a read-it-later app that lets you save and organize everything you read online.

-
-
-

This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities:

-
    -
  • -

    Saving

    -
  • -
  • -

    Reading

    -
  • -
  • -

    Organizing

    -
  • -
  • -

    Integrations

    -
  • -
-

- The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them. -

-

There are five ways to save links to pages or articles that you wish to read later:

-
    -
  • -

    Saving from Your Omnivore Library

    -
  • -
  • -

    Saving from a Browser 

    -
  • -
  • -

    Saving from a Phone or Tablet (iOS or Android)

    -
  • -
  • -

    Newsletter Subscriptions via Email

    -
  • -
  • -

    - Saving PDFs from a Mac
    -

    -
  • -
-

Saving from Your Omnivore Library

-

- 1. In the upper right corner of your Library, tap the Add Link button.
- 2. Enter the URL you wish to save and tap Add Link.
- 3. The link will appear in your Library the next time you refresh it.
-

-

Saving from a Browser

-

1. Download and install the Omnivore extension for your browser:

- -

- 2. Navigate to the page you wish to save and tap the Omnivore button in your browser’s toolbar or Extensions menu.
- 3. Alternatively, you can right-click (command+click on Mac) on any hyperlink and select Save to Omnivore from the menu.
- 4. The link will appear in your Library the next time you refresh it.
-

-

Saving from a Phone or Tablet

-

The best way to save links from your mobile device is via the Omnivore app. You can download the app here:

- -

Once the mobile app is installed:

-
    -
  1. -

    - In your browser, navigate to the page you wish to save and tap the Share button. -

    -
  2. -
  3. -

    - Tap the Omnivore icon in the Share menu. -

    -
  4. -
  5. -

    The link will appear in your Library the next time you refresh it.

    -
  6. -
-

Newsletter Subscriptions via Email

-

- 1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu. -

-

- 2. Tap Create a New Email Address to add a new email address (ex: username-123abc@inbox.omnivore.app) to the list. -

-

3. Click the Copy icon next to the email address.

-

- 4. Navigate to the signup page for the newsletter you wish to subscribe to.
- 5. Paste the Omnivore email address into the signup form. -

-

6. New newsletters will be automatically delivered to your Omnivore inbox.

-

NOTE: If Omnivore receive's an email that does not look like an article, such as a welcome message, or note from the author, it will be forwarded to your Omnivore account email address (the email you registered with).

-

Saving PDFs from a Mac 

-
    -
  1. -

    - Install the Mac App -

    -
  2. -
  3. -

    On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name.

    -
  4. -
  5. -

    - Select Share from the menu and choose Omnivore. -

    -
  6. -
  7. -

    The link will appear in your Library the next time you refresh it.

    -
  8. -
-

Reading

-

Click any link saved in your Library to enter the Reader view. 

-

Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load.

-

While reading, you can:

-
    -
  • -

    Change Formatting

    -
  • -
  • -

    Highlight Text

    -
  • -
  • -

    Add Notes

    -
  • -
  • -

    View All Saved Highlights and Notes

    -
  • -
  • -

    Track Reading Progress

    -
  • -
-

Change Formatting 

-
    -
  1. -

    - Theme: Tap your photo, initial, or avatar  in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme. -

    -
  2. -
  3. -

    - Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing. -

    -
  4. -
-

Highlight Text

-
    -
  1. -

    Select the text you wish to highlight.

    -
  2. -
  3. -

    - Tap the Highlight button. -

    -
  4. -
  5. -

    The text will appear highlighted next time you view the article.

    -
  6. -
-

Add Notes

-
    -
  1. -

    Highlight a section of text where you wish to add a note.

    -
  2. -
  3. -

    - Tap the Note button, type your note, and tap Save. -

    -
  4. -
  5. -

    The Note icon will appear next time you view this article.

    -
  6. -
-

View All Saved Highlights and Notes

-
    -
  1. -

    Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page.

    -
  2. -
  3. -

    To remove a note or highlight, select it from the list and tap the Trash icon.

    -
  4. -
-

Track Reading Progress

-

Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading.

-

Organizing

-

By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions: 

-
    -
  • -

    Archiving

    -
  • -
  • -

    Labels

    -
  • -
  • -

    Search

    -
  • -
  • -

    Filters

    -
  • -
-

Archiving

-
    -
  1. -

    Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu).

    -
  2. -
  3. -

    - Select Archive. -

    -
  4. -
  5. -

    The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below).

    -
  6. -
-

- Labels -

-
    -
  1. -

    - Tap the Menu icon next to any link and select Set Labels. -

    -
  2. -
  3. -

    - Select an existing label from the list or tap Edit Labels to create a new one. -

    -
  4. -
  5. -

    The label will appear next to the link in your Library. Tap it to view all links with the same label.

    -
  6. -
  7. -

    - Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label -

    -
  8. -
  9. -

    Note: Omnivore will automatically assign some labels, such as “Newsletters.”

    -
  10. -
-

Search

-
    -
  1. -

    To search through all your saved links, enter a keyword or phrase in the search bar. 

    -
  2. -
  3. -

    - You can combine keywords with labels and filters to focus your search even further. Learn more about advanced search. -

    -
  4. -
-

Filters

-
    -
  1. -

    - Use the Filters menu to refine your Library view (some filters may be visible by default). -

    -
  2. -
  3. -

    - Select Read Later to view a list of all your non-archived links except Newsletters. -

    -
  4. -
  5. -

    - Select Highlights to view the text selections you have highlighted in all your saved pages.  -

    -
  6. -
  7. -

    - Select Today to view a list of links you saved today. -

    -
  8. -
  9. -

    - Select Newsletters to view links saved via your newsletter subscriptions. -

    -
  10. -
-

Integrations

-

Omnivore allows integrations with knowledge bases and note-taking apps including:

-
    -
  • -

    Logseq

    -
  • -
  • -

    Webhooks

    -
  • -
-

Logseq

-

- With Omnivore's Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide. -

-

Webhooks

-

- Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive. -

-
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_get_started-original.html b/packages/api/src/services/popular_reads/omnivore_get_started-original.html deleted file mode 100644 index b536f4ccd..000000000 --- a/packages/api/src/services/popular_reads/omnivore_get_started-original.html +++ /dev/null @@ -1,1670 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Getting Started with Omnivore - Omnivore - - - - - - - - - - - - - - - - - - - - -
-
- - -
- - - - -
-
-
- -
-
-
-
-
-
-
- -
-
-
- -
-
- Learn the best ways to save links with Omnivore -
-
-
-
- Omnivore -
-
- -
- - - - -
- 2 -
- - - - - -
-
-
-
-
- -
-
- Highlighted <code> in Omnivore -
-
-
-
- Omnivore -
-
- -
- - - - -
- 1 -
- - - - - -
-
-
-
-
- -
-
- Add to your library with your Omnivore email address -
-
-
-
- Omnivore -
-
- -
- - - - -
- 1 -
- - - - - -
-
-
See all - - - - -
-
-
-
-
- -
- -
- -
-
-
- - - - - - - diff --git a/packages/api/src/services/popular_reads/omnivore_ios-content.html b/packages/api/src/services/popular_reads/omnivore_ios-content.html deleted file mode 100644 index 2f79a0603..000000000 --- a/packages/api/src/services/popular_reads/omnivore_ios-content.html +++ /dev/null @@ -1,94 +0,0 @@ -
-
-
-

- With the Omnivore app for iOS, it’s easy to save web pages and articles or archive web content to read later. -

-

- The Omnivore app uses the iOS Share System, which lets you send items from one app (such as Safari) to another (such as Messages or Mail).  -

-
    -
  • -

    Step 1: Log in to the Omnivore app.

    -
  • -
  • -

    Step 2: Add Omnivore to your Share menu favorites.

    -
  • -
  • -

    Step 3: Save links to your Omnivore Library.

    -
  • -
-
-

- -

-
-

You must be logged in before you can save links via the Share menu. If you don’t already have an Omnivore account, you can sign up for free from the login screen.

-

- Note: If you haven’t installed the iOS app, download it here: https://omnivore.app/install/ios -

-

- Step 2: Add Omnivore to your Share menu favorites. -

-

Start by viewing the Share menu from within any supported iOS app (we’ve used Safari for this example).

-
    -
  1. -

    - Tap the Share icon at the bottom of the screen. -

    -
  2. -
  3. -

    - Swipe left to the end of the list of app icons and tap More. -

    -
  4. -
  5. -

    - Tap Edit at the top of the screen. -

    -
  6. -
  7. -

    - Scroll down until you see the Omnivore icon and tap the + icon next to it.  -

    -
  8. -
  9. -

    - Press and hold the three-bar icon and drag Omnivore to one of the top positions under Favorites. Tap Done to close the menu. -

    -
  10. -
  11. -

    - Omnivore will appear as one of the first options the next time you use the Share feature (you may need to restart Safari). -

    -
  12. -
-

- Step 3: Save links to your Omnivore Library -

-

- Start by navigating to the page or article you wish to save. Please note that Omnivore will save the content that appears on your screen (not just a link), so if the page is behind a paywall and you are logged into the paywalled site, you will save the paid content.  -

-
    -
  1. -

    - While viewing the page you’d like to save, tap the Share icon. -

    -
  2. -
  3. -

    - Tap the Omnivore icon in the Share menu. -

    -
  4. -
  5. -

    - Tag the article with one or more labels (optional) and tap Read Now or Read Later -

    -
  6. -
  7. -

    If you choose Read Later, the link will appear in your Library the next time you open the Omnivore app.

    -
  8. -
-
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_ios-original.html b/packages/api/src/services/popular_reads/omnivore_ios-original.html deleted file mode 100644 index 86daf6739..000000000 --- a/packages/api/src/services/popular_reads/omnivore_ios-original.html +++ /dev/null @@ -1,1456 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Saving Links from Your iPhone or iPad - Omnivore - - - - - - - - - - - - - - - - - - - - - -
-
- - -
- - - - -
-
-
- -
-
-
-
-
-
- Comments -
-
-
-
-
- -
-
- -
-
-
-
- - - -
-
-
-
-
-
-
-
- -
-
-
- -
-
- Learn the best ways to save links with Omnivore -
-
-
-
- Omnivore -
-
- -
- - - - -
- 2 -
- - - - - -
-
-
-
-
- -
-
- Highlighted <code> in Omnivore -
-
-
-
- Omnivore -
-
- -
- - - - -
- 1 -
- - - - - -
-
-
-
-
- -
-
- Add to your library with your Omnivore email address -
-
-
-
- Omnivore -
-
- -
- - - - -
- 1 -
- - - - - -
-
-
See all - - - - -
-
-
-
-
- -
- -
- -
-
-
- - - - - - - diff --git a/packages/api/src/services/popular_reads/omnivore_organize-content.html b/packages/api/src/services/popular_reads/omnivore_organize-content.html deleted file mode 100644 index b2ccf3e7c..000000000 --- a/packages/api/src/services/popular_reads/omnivore_organize-content.html +++ /dev/null @@ -1,73 +0,0 @@ -
-
-

Omnivore provides labels (also known as tags) to help you organize your library. Labels can be added to any saved read, and your library can be filtered based on labels.

-

On the web if you have a larger screen you can find the labels tool on the left side of the screen.

-
-
- - - - - -
Adding a label from the left menu
-
-
-

For a smaller screen you will find the labels button at the top of the page.

-
-
- - - - - -
Article Actions at the top of the Omnivore Reader for smaller screens
-
-
-

iOS users can long press on an item in the library or access the labels modal from the menu in the top right of the reader page.

-
-
- - - - - -
Editing labels from the library view on iOS
-
-
-

Label searches on iOS

-

On iOS you can use the label search modal to search for specific labels. This will create an OR search and return all links matching the assigned labels.

-
-
- - - - - -
Using labels to filter your search
-
-
-

Using Advanced Search to filter your library with labels

-

Omnivore's advanced search syntax supports searching for multiple labels using AND and OR clauses. You can also negate a label search to find pages that do not have a certain label.

-

Some examples:

-
    -
  • -

    - label:Newsletter finds all pages that have the label Newsletter -

    -
  • -
  • -

    - label:Cooking,Fitness finds all your pages with either the Cooking or Fitness labels -

    -
  • -
  • -

    - label:Newsletter label:Surfing finds all pages with both the Newsletter and Surfing labels -

    -
  • -
  • -

    label:Coding -label:News finds all pages with the Coding label that do not have the News label

    -
  • -
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_organize-original.html b/packages/api/src/services/popular_reads/omnivore_organize-original.html deleted file mode 100644 index ff3036bc6..000000000 --- a/packages/api/src/services/popular_reads/omnivore_organize-original.html +++ /dev/null @@ -1,1022 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Organize your Omnivore library with labels - Omnivore - - - - - - - - - - - - - - - - - -
-
- - -
- -
-
-
- -
-
-
-
-
-
- -
-
-
- -
-
- Learn the best ways to save links with Omnivore - -
-
-
-
- -
-
- Highlighted <code> in Omnivore - -
-
-
-
- -
-
- Today we are happy to launch our new PDF viewer. It is available in our latest iOS release (1.3.0) and on the web. The new PDF viewer supports… - -
-
See all - - - - -
-
-
-
- -
-
- -
-
- - - - - - - - diff --git a/packages/api/src/services/popular_reads/power_read_it_later-content.html b/packages/api/src/services/popular_reads/power_read_it_later-content.html deleted file mode 100644 index d8366c5f5..000000000 --- a/packages/api/src/services/popular_reads/power_read_it_later-content.html +++ /dev/null @@ -1,239 +0,0 @@ -
-
-
-
-
- -
Image via Nuno Cruz
-
-
-
-

- By Tiago Forte of Forte Labs -

-

At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year.

-

- -

-

This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask.

-

What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place.

-

We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours.

-

The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading.

-

The fact is, the ability to read is becoming a source of competitive advantage in the world.

-

I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time.

-

A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.”

-
-
-

-

Start Building Your Second Brain

-

Subscribe below to learn more about the next cohort of the Building a Second Brain course

-
-
-

Read It. Later.

-

Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar.

-

So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout.

-

On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are:

- -

The app I use, Pocket, adds a button to the Chrome toolbar that looks like this:

-
- -
Chrome toolbar
-
-

- Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. -

-

Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved:

-
-
- -
Mac desktop client
-
-

You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self:

-
-
- -
Web browser “tile” view
-
-

Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces).

-

The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development

-

I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity.

-

But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out.

-

There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with:

-
    -
  1. Increasing consumption of long-form content (which is presumably more substantive)
  2. -
  3. Better filtering
  4. -
-

#1 | Increasing Consumption of Long-Form Content

-

In order to consume good ideas, first you have to consume many ideas.

-

This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation.

-

The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances.

-
-

Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. -

-
-

Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them:

-

1. App performance

-

We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off.

-

If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal.

-

Meanwhile, your snaps and instas refresh at precog-like speeds.

-

Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about.

-

A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off.

-
- -
Effect of ad-blocker on loading times of Boston.com, via NYT -
-
-

Yeah that’s pretty much an eternity in mobile behavior land.

-

2. Matching content with your context

-
-
- -
My Pocket list on iPad
-
-
-

Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse.

-

So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in.

-

Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment.

-

David Allen puts it this way:

-
-

“It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.

-
-

You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓

-

3. Asynchronous reading

-

This is one of the least understood barriers to reading in our fragmented timescape.

-

There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article.

-

The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it.

-

Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished.

-

4. Focus

-

A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it.

-

It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing.

-

Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read.

-
-

Actionable info ≠ Reference info ≠ To Read pile

-
-

Ergo,

-
-

Task manager ≠ Evernote ≠ Pocket

-
-

#2 | Better filtering

-

Now you’ve got the funnel filled. It’s time to narrow it.

-

Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources.

-

The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape.

-

Yeah.

-

In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you.

-

Procrastination is the most powerful force in the universe. It will find a way. -

-

I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time.

-

I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session.

-

So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab.

-

Here’s what a typical Monday morning link-fest looks like, just from email:

-

- -

-

Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing).

-

There’s only one rule: NO READING OR WATCHING! -

-

Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in.

-

I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about.

-

What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me.

-
-

Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide

-
-

Here’s a visual of how this works, from my Pocket analytics:

-

- -

-

You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week.

-

- /sidebar -

-

Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back.

-

I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on.

-

If only this method would scale.

-

- /end_sidebar -

-

Mo’ apps, mo’ problems

-

There are drawbacks, which I’ve glossed over until now. The two main ones:

-

1. Formatting issues

-

Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker.

-

The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed.

-

One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer.

-

2. Dependence

-

Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me.

-

For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved.

-

You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow:

-

- Highlight > Share > Share to Evernote > Save -

-

…has been replaced with this:

-

- Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket -

-

Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date.

-

- /rant_over -

-

Progress Traps and Paradigms

-

The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little.

-

As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know.

-

As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid.

-

What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go?

-

I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.”

-

When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens.

-

Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere.

-

But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on.

-

There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself.

-

Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain.

-

I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms.

-

I can’t put it any better than Donella Meadows, in her seminal piece on complex systems:

-
-

People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. -

-
-
-

It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia.

-
-
-

In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. -

-
-

Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own.

-

The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds.

-

And that is the secret power of Read It Later apps.

-

- P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) -

-
-

Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.

-
-
-

-

Join the Forte Labs Newsletter

-

Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I'll send you my Top 10 All-Time Articles right away as a thank you.

-
-
-
- -
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/power_read_it_later-original.html b/packages/api/src/services/popular_reads/power_read_it_later-original.html deleted file mode 100644 index 69ce2a783..000000000 --- a/packages/api/src/services/popular_reads/power_read_it_later-original.html +++ /dev/null @@ -1,1551 +0,0 @@ - - - - - - - - - - - - The Secret Power of ‘Read It Later’ Apps - Forte Labs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-

- The Secret Power of ‘Read It Later’ Apps -

-
-
- - -
-
-
-

- Estimated reading time: 14 minutes -

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
- Image via Nuno Cruz -
-
-
-
-

- By Tiago Forte of Forte Labs -

-

- At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year. -

-

- -

-

- This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask. -

-

- What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place. -

-

- We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours. -

-

- The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading. -

-

- The fact is, the ability to read is becoming a source of competitive advantage in the world. -

-

- I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time. -

-

- A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.” -

-
-
-
-
-
-

- Start Building Your Second Brain -

-
-
-

- Subscribe below to learn more about the next cohort of the Building a Second Brain course -

-
-
-
-
    -
    -
    - -
    -
    -
    -
    -
    -
    -

    -   -

    -

    - Read It. Later. -

    -

    - Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar. -

    -

    - So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout. -

    -

    - On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are: -

    - -

    - The app I use, Pocket, adds a button to the Chrome toolbar that looks like this: -

    -
    - -
    - Chrome toolbar -
    -
    -

    - Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. -

    -

    - Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved: -

    -
    -
    - -
    - Mac desktop client -
    -
    -

    - You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self: -

    -
    -
    - -
    - Web browser “tile” view -
    -
    -

    - Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces). -

    -

    - The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development -

    -

    - I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity. -

    -

    - But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out. -

    -

    - There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with: -

    -
      -
    1. Increasing consumption of long-form content (which is presumably more substantive) -
    2. -
    3. Better filtering -
    4. -
    -

    - #1 | Increasing Consumption of Long-Form Content -

    -

    - In order to consume good ideas, first you have to consume many ideas. -

    -

    - This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation. -

    -

    - The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances. -

    -
    -

    - Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. -

    -
    -

    - Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them: -

    -

    - 1. App performance -

    -

    - We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off. -

    -

    - If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal. -

    -

    - Meanwhile, your snaps and instas refresh at precog-like speeds. -

    -

    - Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about. -

    -

    - A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off. -

    -
    - -
    - Effect of ad-blocker on loading times of Boston.com, via NYT -
    -
    -

    - Yeah that’s pretty much an eternity in mobile behavior land. -

    -

    - 2. Matching content with your context -

    -
    -
    - -
    - My Pocket list on iPad -
    -
    -
    -

    - Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse. -

    -

    - So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in. -

    -

    - Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment. -

    -

    - David Allen puts it this way: -

    -
    -

    - “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.” -

    -
    -

    - You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓ -

    -

    - 3. Asynchronous reading -

    -

    - This is one of the least understood barriers to reading in our fragmented timescape. -

    -

    - There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article. -

    -

    - The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it. -

    -

    - Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished. -

    -

    - 4. Focus -

    -

    - A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it. -

    -

    - It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing. -

    -

    - Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read. -

    -
    -

    - Actionable info ≠ Reference info ≠ To Read pile -

    -
    -

    - Ergo, -

    -
    -

    - Task manager ≠ Evernote ≠ Pocket -

    -
    -

    - #2 | Better filtering -

    -

    - Now you’ve got the funnel filled. It’s time to narrow it. -

    -

    - Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources. -

    -

    - The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape. -

    -

    - Yeah. -

    -

    - In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you. -

    -

    - Procrastination is the most powerful force in the universe. It will find a way. -

    -

    - I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time. -

    -

    - I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session. -

    -

    - So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab. -

    -

    - Here’s what a typical Monday morning link-fest looks like, just from email: -

    -

    - -

    -

    - Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing). -

    -

    - There’s only one rule: NO READING OR WATCHING! -

    -

    - Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in. -

    -

    - I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about. -

    -

    - What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me. -

    -
    -

    - Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide -

    -
    -

    - Here’s a visual of how this works, from my Pocket analytics: -

    -

    - -

    -

    - You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week. -

    -

    - /sidebar -

    -

    - Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back. -

    -

    - I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on. -

    -

    - If only this method would scale. -

    -

    - /end_sidebar -

    -

    - Mo’ apps, mo’ problems -

    -

    - There are drawbacks, which I’ve glossed over until now. The two main ones: -

    -

    - 1. Formatting issues -

    -

    - Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker. -

    -

    - The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed. -

    -

    - One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer. -

    -

    - 2. Dependence -

    -

    - Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me. -

    -

    - For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved. -

    -

    - You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow: -

    -

    - Highlight > Share > Share to Evernote > Save -

    -

    - …has been replaced with this: -

    -

    - Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket -

    -

    - Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date. -

    -

    - /rant_over -

    -

    - Progress Traps and Paradigms -

    -

    - The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little. -

    -

    - As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know. -

    -

    - As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid. -

    -

    - What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go? -

    -

    - I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.” -

    -

    - When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens. -

    -

    - Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere. -

    -

    - But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on. -

    -

    - There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself. -

    -

    - Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain. -

    -

    - I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms. -

    -

    - I can’t put it any better than Donella Meadows, in her seminal piece on complex systems: -

    -
    -

    - People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. -

    -
    -
    -

    - It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia. -

    -
    -
    -

    - In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. -

    -
    -

    - Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own. -

    -

    - The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds. -

    -

    - And that is the secret power of Read It Later apps. -

    -

    - P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) -

    -
    - Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.
    -
    -
    -
    -
    -
    -
    -
    -

    - Join the Forte Labs Newsletter -

    -
    -
    -

    - Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I'll send you my Top 10 All-Time Articles right away as a thank you. -

    -
    -
    -
    -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - - - -
      -
        -
      • - -
      • -
      • - -
      • -
      • - -
      • -
      • - -
      • -
      • - -
      • -
      • - -
      • -
      -
      -
      - -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/api/src/services/popular_reads/rlove_carnitas-content.html b/packages/api/src/services/popular_reads/rlove_carnitas-content.html deleted file mode 100644 index a02fdb0d6..000000000 --- a/packages/api/src/services/popular_reads/rlove_carnitas-content.html +++ /dev/null @@ -1,34 +0,0 @@ -
      -
      -
      -
      -

      I used to have a bunch of recipes up online. But writing recipes is no fun; it is difficult to capture the beauty of a dish with a bunch of steps. Moreover, using recipes isn’t how I cook. I want to understand the flavors of a dish and then execute it in my own way, in my own hands. So the recipes went away.

      -

      But one of the most popular — and one of my personal favorites — was a recipe for the Mexican pork dish carnitas. It was a fun, relatively easy recipe, not traditional in approach but fairly traditional (and really delicious) in output. Folks keep asking for it. So here it is, in hopes I can eat it at your next house party.

      -

      Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. The traditional recipe is simple: several pounds of pork shoulder, a pound or two of lard, orange peel, and some water (or coca-cola), slow roasted and then “boiled” to a crisp. That is…a bit much. What follows is not an authentic approach.

      -
      -

      -

      -
      Photo by flickr user mccun934 licensed CC BY 2.0 -
      -
      -

      Never ones to eschew taste for health, the French have nonetheless taught us nothing if not that we can make a succulent, flavorful dish without boiling a tough cut of meat in lard. In that vein, my recipe is more of a braiser de porc than a confit de porc — pork shoulder, aromatics, citrus juice, and a little Grand Marnier, slow cooked on the stove. Healthier for the heart, but also — more importantly — tastier to the tongue. To obtain that classic carnitas crisp, we braise the meat uncovered until the liquid evaporates and then move the pot to the oven and caramelize.

      -

      Makes about 6 servings.

      -

      Ingredients:

      -

      3 lbs boneless pork shoulder, preferably Boston butt (the upper shoulder)
      4 tablespoons grape seed oil
      6 cloves garlic, peeled and thinly-sliced
      2 oranges, juiced, plus the zest of 1/2
      2 limes, juiced and zested
      several cups chicken stock
      1 teaspoon ground cayenne pepper
      1 tablespoon ground ancho chile
      1 tablespoon ground chipotle pepper
      2 teaspoons cumin
      1 teaspoon ground cinnamon
      1 teaspoon freshly-ground black pepper
      - herb sachet with 8 sprigs Mexican oregano, 6 sprigs thyme, 2 sticks cinnamon, and 2 bay leaves, wrapped in a cheese cloth and tied shut with cooking twine
      1 cup Grand Marnier
      coarse sea salt, to coat meat, plus more to taste -

      -

      Cut the pork shoulder into 5" chunks. Remove any gratuitously-excessive fat, but leave at least a thin layer. Sprinkle the chunks with sea salt. Let sit at room temperature at least 30 minutes. Pat dry.

      -

      Heat the grape seed oil in a dutch oven over medium-high heat. Sauté the pork shoulder until well browned on each side. If needed, sauté across multiple batches.

      -

      Add the garlic and sauté for one more minute.

      -

      Add the orange zest, lime zest, ground cayenne pepper, ground ancho chile, ground chipotle chile, cumin, ground cinnamon, black pepper, and herb sachet. Mix.

      -

      Add the Grand Marnier, orange juice, and lime juice. Stir, scrapping the bottom of the pot.

      -

      Add chicken stock as needed such that the pork is two-thirds submerged in liquid.

      -

      Stirring, raise heat to high and bring to a boil. Once boiling, lower heat until the liquid is at a simmer and braise, uncovered, stirring occasionally, until the pork is cooked and tender but not disintegrating and the liquid is reduced by at least two-thirds, about three hours. If the liquid gets perilously-low while cooking, add a little chicken stock.

      -

      Remove pork from pot. Once cool enough to handle, use a fork to shred the pork into bite-sized, but fairly large, chunks, removing any fatty pieces as desired.

      -

      Preheat oven to 450°F.

      -

      Return pork chunks to pot. Place uncovered pot in oven. Continue cooking until the liquid has evaporated and the pork is crispy and starting to caramelize, about 20 minutes.

      -

      Taste and adjust salt. Serve with warm corn tortillas, guacamole, pico de gallo, diced white onion, chopped cilantro, lime wedges, margaritas, and college football.

      -
      -
      -
      -
      \ No newline at end of file diff --git a/packages/api/src/services/popular_reads/rlove_carnitas-original.html b/packages/api/src/services/popular_reads/rlove_carnitas-original.html deleted file mode 100644 index e4db70b00..000000000 --- a/packages/api/src/services/popular_reads/rlove_carnitas-original.html +++ /dev/null @@ -1,1230 +0,0 @@ - - - - - Slow-Braised Carnitas Recipe. Robert Love’s recipe for carnitas, in… | by Robert Love | Medium - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      -
      -
      - -
      - -
      -
      -
      - -
      -
      - -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -

      - Slow-Braised Carnitas Recipe -

      -
      - - - -
      -
      - -
      -
      - Photo by flickr user mccun934 licensed CC BY 2.0 -
      -
      - - - - - - - - - - - - - - - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -

      - More from Robert Love -

      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -

      - Google, Android, Linux kernel, author, Boston -

      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -

      - Love podcasts or audiobooks? Learn on the go with our new app. -

      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - - - -
      - -

      - Get the Medium app -

      -
      -
      - A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store -
      A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - - -
      -
      -
      -
      -
      - -
      -
      -
      - Robert Love -
      -
      -
      -

      - Robert Love -

      -
      -
      -

      - Google, Android, Linux kernel, author, Boston -

      -
      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -

      - More from Medium -

      -
      - -
      -

      - This Produce Season: Winter -

      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - - -
      -
      -

      - Eli’s Level 2 Board & Train: Small Dog with Big Dog Syndrome -

      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -

      - in -

      -
      - -
      -
      -

      - Masala Chai Recipe -

      -
      -
      -
      -
      -
      - A cup of masala chai (Indian spiced tea) in a green plate -
      -
      -
      -
      -
      - - - -
      -

      - in -

      -
      - -
      -

      - Vegemite Spaghetti Recipe — An Easy One Pot Dinner -

      -
      -
      -
      -
      - Vegemite pasta spaghetti recipe -
      -
      -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/db/migrations/0140.do.popular_read.sql b/packages/db/migrations/0140.do.popular_read.sql new file mode 100755 index 000000000..1c769c652 --- /dev/null +++ b/packages/db/migrations/0140.do.popular_read.sql @@ -0,0 +1,9931 @@ +-- Type: DO +-- Name: popular_read +-- Description: Create omnivore.popular_read table + +BEGIN; + +CREATE TABLE omnivore.popular_read ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + key text NOT NULL, + original_url text NOT NULL, + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + thumbnail text, + published_at timestamptz, + site_name text NOT NULL, + slug text NOT NULL, + readable_content text NOT NULL, + original_content text NOT NULL, + word_count integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + UNIQUE (key) +); + +CREATE TRIGGER update_popular_read_modtime BEFORE UPDATE ON omnivore.popular_read FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +GRANT SELECT ON omnivore.popular_read TO omnivore_user; + +INSERT INTO omnivore.popular_read (key, original_url, title, author, description, thumbnail, published_at, site_name, slug, readable_content, original_content, word_count) +VALUES + ('omnivore_get_started', 'https://blog.omnivore.app/p/getting-started-with-omnivore', 'Getting Started with Omnivore', 'The Omnivore Team', 'Get the most out of Omnivore by learning how to use it.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sxQnqya1QNApB7ZAGPj9K20AU6sw0UAnjmAIy2ub8hUU/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F658efff4-341a-4720-8cf6-9b2bdbedfaa7_800x668.gif', '2021-10-13', 'Omnivore Blog', 'getting-started-with-omnivore', '
      +
      +
      +

      Omnivore is a read-it-later app that lets you save and organize everything you read online.

      +
      +
      +

      This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities:

      +
        +
      • +

        Saving

        +
      • +
      • +

        Reading

        +
      • +
      • +

        Organizing

        +
      • +
      • +

        Integrations

        +
      • +
      +

      + The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them. +

      +

      There are five ways to save links to pages or articles that you wish to read later:

      +
        +
      • +

        Saving from Your Omnivore Library

        +
      • +
      • +

        Saving from a Browser 

        +
      • +
      • +

        Saving from a Phone or Tablet (iOS or Android)

        +
      • +
      • +

        Newsletter Subscriptions via Email

        +
      • +
      • +

        + Saving PDFs from a Mac
        +

        +
      • +
      +

      Saving from Your Omnivore Library

      +

      + 1. In the upper right corner of your Library, tap the Add Link button.
      + 2. Enter the URL you wish to save and tap Add Link.
      + 3. The link will appear in your Library the next time you refresh it.
      +

      +

      Saving from a Browser

      +

      1. Download and install the Omnivore extension for your browser:

      + +

      + 2. Navigate to the page you wish to save and tap the Omnivore button in your browser’s toolbar or Extensions menu.
      + 3. Alternatively, you can right-click (command+click on Mac) on any hyperlink and select Save to Omnivore from the menu.
      + 4. The link will appear in your Library the next time you refresh it.
      +

      +

      Saving from a Phone or Tablet

      +

      The best way to save links from your mobile device is via the Omnivore app. You can download the app here:

      + +

      Once the mobile app is installed:

      +
        +
      1. +

        + In your browser, navigate to the page you wish to save and tap the Share button. +

        +
      2. +
      3. +

        + Tap the Omnivore icon in the Share menu. +

        +
      4. +
      5. +

        The link will appear in your Library the next time you refresh it.

        +
      6. +
      +

      Newsletter Subscriptions via Email

      +

      + 1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu. +

      +

      + 2. Tap Create a New Email Address to add a new email address (ex: username-123abc@inbox.omnivore.app) to the list. +

      +

      3. Click the Copy icon next to the email address.

      +

      + 4. Navigate to the signup page for the newsletter you wish to subscribe to.
      + 5. Paste the Omnivore email address into the signup form. +

      +

      6. New newsletters will be automatically delivered to your Omnivore inbox.

      +

      NOTE: If Omnivore receive's an email that does not look like an article, such as a welcome message, or note from the author, it will be forwarded to your Omnivore account email address (the email you registered with).

      +

      Saving PDFs from a Mac 

      +
        +
      1. +

        + Install the Mac App +

        +
      2. +
      3. +

        On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name.

        +
      4. +
      5. +

        + Select Share from the menu and choose Omnivore. +

        +
      6. +
      7. +

        The link will appear in your Library the next time you refresh it.

        +
      8. +
      +

      Reading

      +

      Click any link saved in your Library to enter the Reader view. 

      +

      Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load.

      +

      While reading, you can:

      +
        +
      • +

        Change Formatting

        +
      • +
      • +

        Highlight Text

        +
      • +
      • +

        Add Notes

        +
      • +
      • +

        View All Saved Highlights and Notes

        +
      • +
      • +

        Track Reading Progress

        +
      • +
      +

      Change Formatting 

      +
        +
      1. +

        + Theme: Tap your photo, initial, or avatar  in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme. +

        +
      2. +
      3. +

        + Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing. +

        +
      4. +
      +

      Highlight Text

      +
        +
      1. +

        Select the text you wish to highlight.

        +
      2. +
      3. +

        + Tap the Highlight button. +

        +
      4. +
      5. +

        The text will appear highlighted next time you view the article.

        +
      6. +
      +

      Add Notes

      +
        +
      1. +

        Highlight a section of text where you wish to add a note.

        +
      2. +
      3. +

        + Tap the Note button, type your note, and tap Save. +

        +
      4. +
      5. +

        The Note icon will appear next time you view this article.

        +
      6. +
      +

      View All Saved Highlights and Notes

      +
        +
      1. +

        Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page.

        +
      2. +
      3. +

        To remove a note or highlight, select it from the list and tap the Trash icon.

        +
      4. +
      +

      Track Reading Progress

      +

      Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading.

      +

      Organizing

      +

      By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions: 

      +
        +
      • +

        Archiving

        +
      • +
      • +

        Labels

        +
      • +
      • +

        Search

        +
      • +
      • +

        Filters

        +
      • +
      +

      Archiving

      +
        +
      1. +

        Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu).

        +
      2. +
      3. +

        + Select Archive. +

        +
      4. +
      5. +

        The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below).

        +
      6. +
      +

      + Labels +

      +
        +
      1. +

        + Tap the Menu icon next to any link and select Set Labels. +

        +
      2. +
      3. +

        + Select an existing label from the list or tap Edit Labels to create a new one. +

        +
      4. +
      5. +

        The label will appear next to the link in your Library. Tap it to view all links with the same label.

        +
      6. +
      7. +

        + Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label +

        +
      8. +
      9. +

        Note: Omnivore will automatically assign some labels, such as “Newsletters.”

        +
      10. +
      +

      Search

      +
        +
      1. +

        To search through all your saved links, enter a keyword or phrase in the search bar. 

        +
      2. +
      3. +

        + You can combine keywords with labels and filters to focus your search even further. Learn more about advanced search. +

        +
      4. +
      +

      Filters

      +
        +
      1. +

        + Use the Filters menu to refine your Library view (some filters may be visible by default). +

        +
      2. +
      3. +

        + Select Read Later to view a list of all your non-archived links except Newsletters. +

        +
      4. +
      5. +

        + Select Highlights to view the text selections you have highlighted in all your saved pages.  +

        +
      6. +
      7. +

        + Select Today to view a list of links you saved today. +

        +
      8. +
      9. +

        + Select Newsletters to view links saved via your newsletter subscriptions. +

        +
      10. +
      +

      Integrations

      +

      Omnivore allows integrations with knowledge bases and note-taking apps including:

      +
        +
      • +

        Logseq

        +
      • +
      • +

        Webhooks

        +
      • +
      +

      Logseq

      +

      + With Omnivore''s Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide. +

      +

      Webhooks

      +

      + Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive. +

      +
      +
      +
      +', ' + + + + + + + + + + + + + + + + + + + + + + + + Getting Started with Omnivore - Omnivore + + + + + + + + + + + + + + + + + + + + +
      +
      + + +
      + + + + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 2 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Add to your library with your Omnivore email address +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      See all + + + + +
      +
      +
      +
      +
      + +
      + +
      + +
      +
      +
      + + + + + + + +', 1155), + ('omnivore_ios', 'https://blog.omnivore.app/p/saving-links-from-your-iphone-or', 'Saving Links from Your iPhone or iPad', 'Omnivore', 'Learn how to save articles on iOS.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sWDfv7sARTIdAlx6Rw_6t-QwL3T9aniEJRa1-jVaglNg/https://substackcdn.com/image/youtube/w_728,c_limit/k6RkIqepAig', '2021-10-19', 'Omnivore Blog', 'saving-links-from-your-i-phone-or-i-pad', '
      +
      +
      +

      + With the Omnivore app for iOS, it’s easy to save web pages and articles or archive web content to read later. +

      +

      + The Omnivore app uses the iOS Share System, which lets you send items from one app (such as Safari) to another (such as Messages or Mail).  +

      +
        +
      • +

        Step 1: Log in to the Omnivore app.

        +
      • +
      • +

        Step 2: Add Omnivore to your Share menu favorites.

        +
      • +
      • +

        Step 3: Save links to your Omnivore Library.

        +
      • +
      +
      +

      + +

      +
      +

      You must be logged in before you can save links via the Share menu. If you don’t already have an Omnivore account, you can sign up for free from the login screen.

      +

      + Note: If you haven’t installed the iOS app, download it here: https://omnivore.app/install/ios +

      +

      + Step 2: Add Omnivore to your Share menu favorites. +

      +

      Start by viewing the Share menu from within any supported iOS app (we’ve used Safari for this example).

      +
        +
      1. +

        + Tap the Share icon at the bottom of the screen. +

        +
      2. +
      3. +

        + Swipe left to the end of the list of app icons and tap More. +

        +
      4. +
      5. +

        + Tap Edit at the top of the screen. +

        +
      6. +
      7. +

        + Scroll down until you see the Omnivore icon and tap the + icon next to it.  +

        +
      8. +
      9. +

        + Press and hold the three-bar icon and drag Omnivore to one of the top positions under Favorites. Tap Done to close the menu. +

        +
      10. +
      11. +

        + Omnivore will appear as one of the first options the next time you use the Share feature (you may need to restart Safari). +

        +
      12. +
      +

      + Step 3: Save links to your Omnivore Library +

      +

      + Start by navigating to the page or article you wish to save. Please note that Omnivore will save the content that appears on your screen (not just a link), so if the page is behind a paywall and you are logged into the paywalled site, you will save the paid content.  +

      +
        +
      1. +

        + While viewing the page you’d like to save, tap the Share icon. +

        +
      2. +
      3. +

        + Tap the Omnivore icon in the Share menu. +

        +
      4. +
      5. +

        + Tag the article with one or more labels (optional) and tap Read Now or Read Later +

        +
      6. +
      7. +

        If you choose Read Later, the link will appear in your Library the next time you open the Omnivore app.

        +
      8. +
      +
      +
      +
      ', ' + + + + + + + + + + + + + + + + + + + + + + + + Saving Links from Your iPhone or iPad - Omnivore + + + + + + + + + + + + + + + + + + + + + +
      +
      + + +
      + + + + +
      +
      +
      + +
      +
      +
      +
      +
      +
      + Comments +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      + + + +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 2 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Add to your library with your Omnivore email address +
      +
      +
      +
      + Omnivore +
      +
      + +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      See all + + + + +
      +
      +
      +
      +
      + +
      + +
      + +
      +
      +
      + + + + + + + +', 371), + ('omnivore_organize', 'https://blog.omnivore.app/p/organize-your-omnivore-library-with', 'Organize your Omnivore library with labels', 'The Omnivore Team', 'Use labels to organize your Omnivore library.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sTgJ5Q0XIg_EHdmPWcxtXFmkjn8T6hkJt7S9ziClagYo/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fdaf07af7-5cdb-4ecc-aace-1a46de3e9c58_1827x1090.png', '2022-04-18', 'Omnivore Blog', 'organize-your-omnivore-library-with-labels', '
      +
      +

      Omnivore provides labels (also known as tags) to help you organize your library. Labels can be added to any saved read, and your library can be filtered based on labels.

      +

      On the web if you have a larger screen you can find the labels tool on the left side of the screen.

      +
      +
      + + + + + +
      Adding a label from the left menu
      +
      +
      +

      For a smaller screen you will find the labels button at the top of the page.

      +
      +
      + + + + + +
      Article Actions at the top of the Omnivore Reader for smaller screens
      +
      +
      +

      iOS users can long press on an item in the library or access the labels modal from the menu in the top right of the reader page.

      +
      +
      + + + + + +
      Editing labels from the library view on iOS
      +
      +
      +

      Label searches on iOS

      +

      On iOS you can use the label search modal to search for specific labels. This will create an OR search and return all links matching the assigned labels.

      +
      +
      + + + + + +
      Using labels to filter your search
      +
      +
      +

      Using Advanced Search to filter your library with labels

      +

      Omnivore''s advanced search syntax supports searching for multiple labels using AND and OR clauses. You can also negate a label search to find pages that do not have a certain label.

      +

      Some examples:

      +
        +
      • +

        + label:Newsletter finds all pages that have the label Newsletter +

        +
      • +
      • +

        + label:Cooking,Fitness finds all your pages with either the Cooking or Fitness labels +

        +
      • +
      • +

        + label:Newsletter label:Surfing finds all pages with both the Newsletter and Surfing labels +

        +
      • +
      • +

        label:Coding -label:News finds all pages with the Coding label that do not have the News label

        +
      • +
      +
      +
      +', ' + + + + + + + + + + + + + + + + + + + + + + + + Organize your Omnivore library with labels - Omnivore + + + + + + + + + + + + + + + + + +
      +
      + + +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore + +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore + +
      +
      +
      +
      + +
      +
      + Today we are happy to launch our new PDF viewer. It is available in our latest iOS release (1.3.0) and on the web. The new PDF viewer supports… + +
      +
      See all + + + + +
      +
      +
      +
      + +
      +
      + +
      +
      + + + + + + + + +', 259), + ('rlove_carnitas', 'https://medium.com/@rlove/carnitas-ff0ef1044ae9', 'Slow-Braised Carnitas Recipe', 'Robert Love', 'Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. This is not an authentic recipe.', 'https://proxy-prod.omnivore-image-cache.app/88x88,sIcDXt3Ar0baKG1e1Yi1e2VUZFL85xPlOeEfAxF-s-Nw/https://miro.medium.com/max/1200/1*Wl-dMBJpSgPUxUOnPQthyg.jpeg', '2017-02-24', '@rlove', 'slow-braised-carnitas-recipe', '
      +
      +
      +
      +

      I used to have a bunch of recipes up online. But writing recipes is no fun; it is difficult to capture the beauty of a dish with a bunch of steps. Moreover, using recipes isn’t how I cook. I want to understand the flavors of a dish and then execute it in my own way, in my own hands. So the recipes went away.

      +

      But one of the most popular — and one of my personal favorites — was a recipe for the Mexican pork dish carnitas. It was a fun, relatively easy recipe, not traditional in approach but fairly traditional (and really delicious) in output. Folks keep asking for it. So here it is, in hopes I can eat it at your next house party.

      +

      Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. The traditional recipe is simple: several pounds of pork shoulder, a pound or two of lard, orange peel, and some water (or coca-cola), slow roasted and then “boiled” to a crisp. That is…a bit much. What follows is not an authentic approach.

      +
      +

      +

      +
      Photo by flickr user mccun934 licensed CC BY 2.0 +
      +
      +

      Never ones to eschew taste for health, the French have nonetheless taught us nothing if not that we can make a succulent, flavorful dish without boiling a tough cut of meat in lard. In that vein, my recipe is more of a braiser de porc than a confit de porc — pork shoulder, aromatics, citrus juice, and a little Grand Marnier, slow cooked on the stove. Healthier for the heart, but also — more importantly — tastier to the tongue. To obtain that classic carnitas crisp, we braise the meat uncovered until the liquid evaporates and then move the pot to the oven and caramelize.

      +

      Makes about 6 servings.

      +

      Ingredients:

      +

      3 lbs boneless pork shoulder, preferably Boston butt (the upper shoulder)
      4 tablespoons grape seed oil
      6 cloves garlic, peeled and thinly-sliced
      2 oranges, juiced, plus the zest of 1/2
      2 limes, juiced and zested
      several cups chicken stock
      1 teaspoon ground cayenne pepper
      1 tablespoon ground ancho chile
      1 tablespoon ground chipotle pepper
      2 teaspoons cumin
      1 teaspoon ground cinnamon
      1 teaspoon freshly-ground black pepper
      + herb sachet with 8 sprigs Mexican oregano, 6 sprigs thyme, 2 sticks cinnamon, and 2 bay leaves, wrapped in a cheese cloth and tied shut with cooking twine
      1 cup Grand Marnier
      coarse sea salt, to coat meat, plus more to taste +

      +

      Cut the pork shoulder into 5" chunks. Remove any gratuitously-excessive fat, but leave at least a thin layer. Sprinkle the chunks with sea salt. Let sit at room temperature at least 30 minutes. Pat dry.

      +

      Heat the grape seed oil in a dutch oven over medium-high heat. Sauté the pork shoulder until well browned on each side. If needed, sauté across multiple batches.

      +

      Add the garlic and sauté for one more minute.

      +

      Add the orange zest, lime zest, ground cayenne pepper, ground ancho chile, ground chipotle chile, cumin, ground cinnamon, black pepper, and herb sachet. Mix.

      +

      Add the Grand Marnier, orange juice, and lime juice. Stir, scrapping the bottom of the pot.

      +

      Add chicken stock as needed such that the pork is two-thirds submerged in liquid.

      +

      Stirring, raise heat to high and bring to a boil. Once boiling, lower heat until the liquid is at a simmer and braise, uncovered, stirring occasionally, until the pork is cooked and tender but not disintegrating and the liquid is reduced by at least two-thirds, about three hours. If the liquid gets perilously-low while cooking, add a little chicken stock.

      +

      Remove pork from pot. Once cool enough to handle, use a fork to shred the pork into bite-sized, but fairly large, chunks, removing any fatty pieces as desired.

      +

      Preheat oven to 450°F.

      +

      Return pork chunks to pot. Place uncovered pot in oven. Continue cooking until the liquid has evaporated and the pork is crispy and starting to caramelize, about 20 minutes.

      +

      Taste and adjust salt. Serve with warm corn tortillas, guacamole, pico de gallo, diced white onion, chopped cilantro, lime wedges, margaritas, and college football.

      +
      +
      +
      +
      ', ' + + + + Slow-Braised Carnitas Recipe. Robert Love’s recipe for carnitas, in… | by Robert Love | Medium + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      +
      + +
      + +
      +
      +
      + +
      +
      + +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +

      + Slow-Braised Carnitas Recipe +

      +
      + + + +
      +
      + +
      +
      + Photo by flickr user mccun934 licensed CC BY 2.0 +
      +
      + + + + + + + + + + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + More from Robert Love +

      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +

      + Google, Android, Linux kernel, author, Boston +

      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + Love podcasts or audiobooks? Learn on the go with our new app. +

      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + + + +
      + +

      + Get the Medium app +

      +
      +
      + A button that says ''Download on the App Store'', and if clicked it will lead you to the iOS App store +
      A button that says ''Get it on, Google Play'', and if clicked it will lead you to the Google Play store +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + + +
      +
      +
      +
      +
      + +
      +
      +
      + Robert Love +
      +
      +
      +

      + Robert Love +

      +
      +
      +

      + Google, Android, Linux kernel, author, Boston +

      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +

      + More from Medium +

      +
      + +
      +

      + This Produce Season: Winter +

      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + + +
      +
      +

      + Eli’s Level 2 Board & Train: Small Dog with Big Dog Syndrome +

      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +

      + in +

      +
      + +
      +
      +

      + Masala Chai Recipe +

      +
      +
      +
      +
      +
      + A cup of masala chai (Indian spiced tea) in a green plate +
      +
      +
      +
      +
      + + + +
      +

      + in +

      +
      + +
      +

      + Vegemite Spaghetti Recipe — An Easy One Pot Dinner +

      +
      +
      +
      +
      + Vegemite pasta spaghetti recipe +
      +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +', 705), + ('power_read_it_later', 'https://fortelabs.co/blog/the-secret-power-of-read-it-later-apps', 'The Secret Power of ‘Read It Later’ Apps', 'Tiago Forte', 'At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket', 'https://proxy-prod.omnivore-image-cache.app/320x320,sGN5R34M5z068QMXDZD32CQD6mCbxc47hWXm__JVUePE/https://fortelabs.com/wp-content/uploads/2015/11/1rPXwIczUJRCE54v8FfAHGw.jpeg', '2022-01-24', 'Forte Labs', 'the-secret-power-of-read-it-later-apps', '
      +
      +
      +
      +
      + +
      Image via Nuno Cruz
      +
      +
      +
      +

      + By Tiago Forte of Forte Labs +

      +

      At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year.

      +

      + +

      +

      This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask.

      +

      What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place.

      +

      We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours.

      +

      The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading.

      +

      The fact is, the ability to read is becoming a source of competitive advantage in the world.

      +

      I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time.

      +

      A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.”

      +
      +
      +

      +

      Start Building Your Second Brain

      +

      Subscribe below to learn more about the next cohort of the Building a Second Brain course

      +
      +
      +

      Read It. Later.

      +

      Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar.

      +

      So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout.

      +

      On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are:

      + +

      The app I use, Pocket, adds a button to the Chrome toolbar that looks like this:

      +
      + +
      Chrome toolbar
      +
      +

      + Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. +

      +

      Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved:

      +
      +
      + +
      Mac desktop client
      +
      +

      You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self:

      +
      +
      + +
      Web browser “tile” view
      +
      +

      Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces).

      +

      The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development

      +

      I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity.

      +

      But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out.

      +

      There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with:

      +
        +
      1. Increasing consumption of long-form content (which is presumably more substantive)
      2. +
      3. Better filtering
      4. +
      +

      #1 | Increasing Consumption of Long-Form Content

      +

      In order to consume good ideas, first you have to consume many ideas.

      +

      This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation.

      +

      The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances.

      +
      +

      Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. +

      +
      +

      Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them:

      +

      1. App performance

      +

      We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off.

      +

      If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal.

      +

      Meanwhile, your snaps and instas refresh at precog-like speeds.

      +

      Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about.

      +

      A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off.

      +
      + +
      Effect of ad-blocker on loading times of Boston.com, via NYT +
      +
      +

      Yeah that’s pretty much an eternity in mobile behavior land.

      +

      2. Matching content with your context

      +
      +
      + +
      My Pocket list on iPad
      +
      +
      +

      Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse.

      +

      So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in.

      +

      Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment.

      +

      David Allen puts it this way:

      +
      +

      “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.

      +
      +

      You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓

      +

      3. Asynchronous reading

      +

      This is one of the least understood barriers to reading in our fragmented timescape.

      +

      There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article.

      +

      The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it.

      +

      Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished.

      +

      4. Focus

      +

      A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it.

      +

      It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing.

      +

      Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read.

      +
      +

      Actionable info ≠ Reference info ≠ To Read pile

      +
      +

      Ergo,

      +
      +

      Task manager ≠ Evernote ≠ Pocket

      +
      +

      #2 | Better filtering

      +

      Now you’ve got the funnel filled. It’s time to narrow it.

      +

      Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources.

      +

      The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape.

      +

      Yeah.

      +

      In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you.

      +

      Procrastination is the most powerful force in the universe. It will find a way. +

      +

      I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time.

      +

      I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session.

      +

      So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab.

      +

      Here’s what a typical Monday morning link-fest looks like, just from email:

      +

      + +

      +

      Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing).

      +

      There’s only one rule: NO READING OR WATCHING! +

      +

      Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in.

      +

      I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about.

      +

      What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me.

      +
      +

      Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide

      +
      +

      Here’s a visual of how this works, from my Pocket analytics:

      +

      + +

      +

      You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week.

      +

      + /sidebar +

      +

      Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back.

      +

      I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on.

      +

      If only this method would scale.

      +

      + /end_sidebar +

      +

      Mo’ apps, mo’ problems

      +

      There are drawbacks, which I’ve glossed over until now. The two main ones:

      +

      1. Formatting issues

      +

      Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker.

      +

      The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed.

      +

      One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer.

      +

      2. Dependence

      +

      Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me.

      +

      For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved.

      +

      You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow:

      +

      + Highlight > Share > Share to Evernote > Save +

      +

      …has been replaced with this:

      +

      + Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket +

      +

      Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date.

      +

      + /rant_over +

      +

      Progress Traps and Paradigms

      +

      The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little.

      +

      As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know.

      +

      As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid.

      +

      What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go?

      +

      I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.”

      +

      When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens.

      +

      Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere.

      +

      But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on.

      +

      There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself.

      +

      Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain.

      +

      I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms.

      +

      I can’t put it any better than Donella Meadows, in her seminal piece on complex systems:

      +
      +

      People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. +

      +
      +
      +

      It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia.

      +
      +
      +

      In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. +

      +
      +

      Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own.

      +

      The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds.

      +

      And that is the secret power of Read It Later apps.

      +

      + P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) +

      +
      +

      Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.

      +
      +
      +

      +

      Join the Forte Labs Newsletter

      +

      Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I''ll send you my Top 10 All-Time Articles right away as a thank you.

      +
      +
      +
      + +
      +
      +', ' + + + + + + + + + + + The Secret Power of ‘Read It Later’ Apps - Forte Labs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + The Secret Power of ‘Read It Later’ Apps +

      +
      +
      + + +
      +
      +
      +

      + Estimated reading time: 14 minutes +

      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      + Image via Nuno Cruz +
      +
      +
      +
      +

      + By Tiago Forte of Forte Labs +

      +

      + At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year. +

      +

      + +

      +

      + This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask. +

      +

      + What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place. +

      +

      + We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours. +

      +

      + The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading. +

      +

      + The fact is, the ability to read is becoming a source of competitive advantage in the world. +

      +

      + I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time. +

      +

      + A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.” +

      +
      +
      +
      +
      +
      +

      + Start Building Your Second Brain +

      +
      +
      +

      + Subscribe below to learn more about the next cohort of the Building a Second Brain course +

      +
      +
      +
      +
        +
        +
        + +
        +
        +
        +
        +
        +
        +

        +   +

        +

        + Read It. Later. +

        +

        + Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar. +

        +

        + So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout. +

        +

        + On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are: +

        + +

        + The app I use, Pocket, adds a button to the Chrome toolbar that looks like this: +

        +
        + +
        + Chrome toolbar +
        +
        +

        + Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. +

        +

        + Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved: +

        +
        +
        + +
        + Mac desktop client +
        +
        +

        + You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self: +

        +
        +
        + +
        + Web browser “tile” view +
        +
        +

        + Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces). +

        +

        + The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development +

        +

        + I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity. +

        +

        + But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out. +

        +

        + There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with: +

        +
          +
        1. Increasing consumption of long-form content (which is presumably more substantive) +
        2. +
        3. Better filtering +
        4. +
        +

        + #1 | Increasing Consumption of Long-Form Content +

        +

        + In order to consume good ideas, first you have to consume many ideas. +

        +

        + This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation. +

        +

        + The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances. +

        +
        +

        + Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. +

        +
        +

        + Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them: +

        +

        + 1. App performance +

        +

        + We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off. +

        +

        + If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal. +

        +

        + Meanwhile, your snaps and instas refresh at precog-like speeds. +

        +

        + Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about. +

        +

        + A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off. +

        +
        + +
        + Effect of ad-blocker on loading times of Boston.com, via NYT +
        +
        +

        + Yeah that’s pretty much an eternity in mobile behavior land. +

        +

        + 2. Matching content with your context +

        +
        +
        + +
        + My Pocket list on iPad +
        +
        +
        +

        + Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse. +

        +

        + So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in. +

        +

        + Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment. +

        +

        + David Allen puts it this way: +

        +
        +

        + “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.” +

        +
        +

        + You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓ +

        +

        + 3. Asynchronous reading +

        +

        + This is one of the least understood barriers to reading in our fragmented timescape. +

        +

        + There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article. +

        +

        + The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it. +

        +

        + Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished. +

        +

        + 4. Focus +

        +

        + A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it. +

        +

        + It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing. +

        +

        + Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read. +

        +
        +

        + Actionable info ≠ Reference info ≠ To Read pile +

        +
        +

        + Ergo, +

        +
        +

        + Task manager ≠ Evernote ≠ Pocket +

        +
        +

        + #2 | Better filtering +

        +

        + Now you’ve got the funnel filled. It’s time to narrow it. +

        +

        + Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources. +

        +

        + The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape. +

        +

        + Yeah. +

        +

        + In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you. +

        +

        + Procrastination is the most powerful force in the universe. It will find a way. +

        +

        + I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time. +

        +

        + I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session. +

        +

        + So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab. +

        +

        + Here’s what a typical Monday morning link-fest looks like, just from email: +

        +

        + +

        +

        + Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing). +

        +

        + There’s only one rule: NO READING OR WATCHING! +

        +

        + Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in. +

        +

        + I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about. +

        +

        + What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me. +

        +
        +

        + Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide +

        +
        +

        + Here’s a visual of how this works, from my Pocket analytics: +

        +

        + +

        +

        + You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week. +

        +

        + /sidebar +

        +

        + Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back. +

        +

        + I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on. +

        +

        + If only this method would scale. +

        +

        + /end_sidebar +

        +

        + Mo’ apps, mo’ problems +

        +

        + There are drawbacks, which I’ve glossed over until now. The two main ones: +

        +

        + 1. Formatting issues +

        +

        + Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker. +

        +

        + The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed. +

        +

        + One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer. +

        +

        + 2. Dependence +

        +

        + Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me. +

        +

        + For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved. +

        +

        + You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow: +

        +

        + Highlight > Share > Share to Evernote > Save +

        +

        + …has been replaced with this: +

        +

        + Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket +

        +

        + Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date. +

        +

        + /rant_over +

        +

        + Progress Traps and Paradigms +

        +

        + The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little. +

        +

        + As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know. +

        +

        + As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid. +

        +

        + What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go? +

        +

        + I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.” +

        +

        + When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens. +

        +

        + Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere. +

        +

        + But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on. +

        +

        + There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself. +

        +

        + Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain. +

        +

        + I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms. +

        +

        + I can’t put it any better than Donella Meadows, in her seminal piece on complex systems: +

        +
        +

        + People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. +

        +
        +
        +

        + It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia. +

        +
        +
        +

        + In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. +

        +
        +

        + Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own. +

        +

        + The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds. +

        +

        + And that is the secret power of Read It Later apps. +

        +

        + P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) +

        +
        + Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.
        +
        +
        +
        +
        +
        +
        +
        +

        + Join the Forte Labs Newsletter +

        +
        +
        +

        + Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I''ll send you my Top 10 All-Time Articles right away as a thank you. +

        +
        +
        +
        +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + +
          +
            +
          • + +
          • +
          • + +
          • +
          • + +
          • +
          • + +
          • +
          • + +
          • +
          • + +
          • +
          +
          +
          + +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +', 3726), + ('elad_meetings', 'http://blog.eladgil.com/2018/07/meeting-etiquette.html', 'Better Meetings', 'Elad Gil', 'How to make meetings more productive.', null, '2018-07-02', 'Elad Blog', 'better-meetings', '
          +
          +

          As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
          +
          1. Determine who is necessary in the meeting.
          + Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
          +
          2. Send out an agenda in advance.
          + What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
          +
          Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
          +
          3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
          + If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
          +
          If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
          +
          4. Kick off the meeting with objectives.
          + Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
          +
          5. Assign a note taker.
          + Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
          +
          6. Send out meeting notes.
          + Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
          +
          Meeting notes would optimally include:
          + a. Subject/topic of meeting
          + b. Date
          + c. Attendees
          + d. Actions/decisions
          + e. Agenda
          + f. Detailed notes
          +
          7. Clean up the meeting calendar ongoing.

          + Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
          +
          NOTES

          + [1] There may be legal reasons to not take notes in specific meetings. Consult with the company''s GC or external counsel on training for meeting note etiquette. +

          +

          MY BOOK +

          +

          You can pre-order the High Growth Handbook here.
          +
          +

          +
          +
          +', ' + + + + + + + + + + + + + + + + + Elad Blog: Better Meetings + + + + + + + + + + + + + + + + + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +

          + Monday, July 2, 2018 +

          +
          +
          +
          + + +

          + Better Meetings +

          +
          +
          +
          +
          +
          + As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
          +
          1. Determine who is necessary in the meeting.
          + Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
          +
          2. Send out an agenda in advance.
          + What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
          +
          Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
          +
          3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
          + If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
          +
          If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
          +
          4. Kick off the meeting with objectives.
          + Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
          +
          5. Assign a note taker.
          + Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
          +
          6. Send out meeting notes.
          + Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
          +
          Meeting notes would optimally include:
          + a. Subject/topic of meeting
          + b. Date
          + c. Attendees
          + d. Actions/decisions
          + e. Agenda
          + f. Detailed notes
          +
          7. Clean up the meeting calendar ongoing.

          + Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
          +
          NOTES

          + [1] There may be legal reasons to not take notes in specific meetings. Consult with the company''s GC or external counsel on training for meeting note etiquette.
          +
          +
          +
          +
          + MY BOOK +
          You can pre-order the High Growth Handbook here.
          +

          +
          +
          +
          +
          + RELATED POSTS +
          + + + + + + +
          +
          +
          + +
          +
          + +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + + + + +', 545), + ('jonbo_digital_tools', 'https://jon.bo/posts/digital-tools/', 'Digital Tools I Wish Existed', 'Jonathan Borichevskiy', 'My digital life in a nutshell: I discover relevant content I don’t have time to consume...', null, '2019-11-28', 'JON.BO', 'digital-tools-i-wish-existed', '
          +
          +

          My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later.

          +

          There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed:

          +
            +
          • +

            The more seamless the acquisition & ingestion, the more engaged I am with the content

            +
          • +
          • +

            Insights are just as likely to be found in a 400-page book as in a 40-minute podcast

            +
          • +
          • +

            Notes and their subsequent review are essential for long-term retention

            +
          • +
          • +

            Recommendations from other humans are as good, if not better, than algorithmic suggestions

            +
          • +
          +

          In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here.

          +

          Queue management for inbound digital content # +

          +

          Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance:

          +
            +
          • +

            Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more.

            +
          • +
          • +

            Every book, article, post, or tweet has the potential to lead to more content.

            +
          • +
          • +

            Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs.

            +
          • +
          • +

            I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee?

            +
          • +
          • +

            Learning, work, news, and entertainment all have different priorities in my life (roughly in that order).

            +
          • +
          • +

            I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month.

            +
          • +
          • +

            I’m not always connected to a stable internet connection.

            +
          • +
          • +

            If it’s a long piece of content I want my position saved reliably so I can resume at a later point.

            +
          • +
          • +

            I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation.

            +
          • +
          • +

            I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place.

            +
          • +
          • +

            I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat.

            +
          • +
          +
          + +
          Relevant XKCD, as is tradition
          +
          +

          Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one.

          +

          Honorable Mentions: Pocket, Instapaper +

          +

          A universal book log, recommendation & sharing system # +

          +

          I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof.

          +

          Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry.

          +

          Whatever this tool for managing my ever-growing reading list will be, it should:

          +
            +
          • +

            Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests.

            +
          • +
          • +

            Allow me to tag books instead of placing them into static lists (think clusters or tag clouds).

            +
          • +
          • +

            Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to.

            +
          • +
          • +

            Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it.

            +
          • +
          • +

            Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was.

            +
          • +
          • +

            Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on.

            +
          • +
          • +

            Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from.

            +
          • +
          +

          Honorable Mentions: None :(

          +

          Intelligent PDF viewers, eBook readers, audiobook & podcast players # +

          +
          + +
          Functionality I want in my document reader
          +
          +

          Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to:

          +
            +
          • +

            Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them.

            +
          • +
          • +

            See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition.

            +
          • +
          • +

            View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know!

            +
          • +
          • +

            Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019.

            +
          • +
          • +

            Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues.

            +
          • +
          • +

            Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points).

            +
          • +
          • +

            Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading.

            +
          • +
          +
          + +
          What I want my audiobook player to look like
          +
          +

          Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could:

          +
            +
          • +

            Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street.

            +
          • +
          • +

            View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps?

            +
          • +
          +

          Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar +

          +

          A centralized search interface for my digital brain (memex) # +

          +

          I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything.

          +

          I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should:

          +
            +
          • +

            Accept and parse the following queries:

            +
              +
            • +

              spacex announcement type:video 2016

              +
            • +
            • +

              links from:jon@test.org topic:python

              +
            • +
            • +

              paper on temperature, productivity referenced in book:Uninhabitable Earth

              +
            • +
            • +

              type:pdf habits digital interfaces

              +
            • +
            • +

              reading comprehension type:blog post

              +
            • +
            • +

              printer ink receipt

              +
            • +
            • +

              type:book read:2017 finance

              +
            • +
            • +

              file:py datetime parse

              +
            • +
            +
          • +
          • +

            Respect my privacy: hosted on something I control and never mined for ads.

            +
          • +
          • +

            Support all my devices with two-way sync so I can search and add to it wherever I am.

            +
          • +
          • +

            Be extensible: allow me to easily ingest my own information and extend with desired functionality.

            +
          • +
          • +

            Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of.

            +
          • +
          • +

            Notify me about changes to documents and webpages I’ve visited.

            +
          • +
          • +

            Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it.

            +
          • +
          +

          Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop +

          +

          Parting Thoughts # +

          +

          I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools.

          +

          It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share.

          +

          I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback.

          +

          + HN Discussion +

          +

          Appendix # +

          +

          + The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS +

          +
          +
          ', ' + + + + Digital Tools I Wish Existed :: up & to the right — Jonathan Borichevskiy + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + + +
          +
          +
          +

          + Digital Tools I Wish Existed +

          +

          + +
          +

          + My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later. +

          +

          + There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed: +

          +
            +
          • +

            + The more seamless the acquisition & ingestion, the more engaged I am with the content +

            +
          • +
          • +

            + Insights are just as likely to be found in a 400-page book as in a 40-minute podcast +

            +
          • +
          • +

            + Notes and their subsequent review are essential for long-term retention +

            +
          • +
          • +

            + Recommendations from other humans are as good, if not better, than algorithmic suggestions +

            +
          • +
          +

          + In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here. +

          +

          + Queue management for inbound digital content # +

          +

          + Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance: +

          +
            +
          • +

            + Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more. +

            +
          • +
          • +

            + Every book, article, post, or tweet has the potential to lead to more content. +

            +
          • +
          • +

            + Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs. +

            +
          • +
          • +

            + I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee? +

            +
          • +
          • +

            + Learning, work, news, and entertainment all have different priorities in my life (roughly in that order). +

            +
          • +
          • +

            + I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month. +

            +
          • +
          • +

            + I’m not always connected to a stable internet connection. +

            +
          • +
          • +

            + If it’s a long piece of content I want my position saved reliably so I can resume at a later point. +

            +
          • +
          • +

            + I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation. +

            +
          • +
          • +

            + I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place. +

            +
          • +
          • +

            + I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat. +

            +
          • +
          +
          + +
          + Relevant XKCD, as is tradition +
          +
          +

          + Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one. +

          +

          + Honorable Mentions: Pocket, Instapaper +

          +

          + A universal book log, recommendation & sharing system # +

          +

          + I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof. +

          +

          + Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry. +

          +

          + Whatever this tool for managing my ever-growing reading list will be, it should: +

          +
            +
          • +

            + Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests. +

            +
          • +
          • +

            + Allow me to tag books instead of placing them into static lists (think clusters or tag clouds). +

            +
          • +
          • +

            + Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to. +

            +
          • +
          • +

            + Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it. +

            +
          • +
          • +

            + Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was. +

            +
          • +
          • +

            + Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on. +

            +
          • +
          • +

            + Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from. +

            +
          • +
          +

          + Honorable Mentions: None :( +

          +

          + Intelligent PDF viewers, eBook readers, audiobook & podcast players # +

          +
          + +
          + Functionality I want in my document reader +
          +
          +

          + Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to: +

          +
            +
          • +

            + Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them. +

            +
          • +
          • +

            + See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition. +

            +
          • +
          • +

            + View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know! +

            +
          • +
          • +

            + Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019. +

            +
          • +
          • +

            + Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues. +

            +
          • +
          • +

            + Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points). +

            +
          • +
          • +

            + Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading. +

            +
          • +
          +
          + +
          + What I want my audiobook player to look like +
          +
          +

          + Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could: +

          +
            +
          • +

            + Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street. +

            +
          • +
          • +

            + View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps? +

            +
          • +
          +

          + Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar +

          +

          + A centralized search interface for my digital brain (memex) # +

          +

          + I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything. +

          +

          + I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should: +

          +
            +
          • +

            + Accept and parse the following queries: +

            +
              +
            • +

              + spacex announcement type:video 2016 +

              +
            • +
            • +

              + links from:jon@test.org topic:python +

              +
            • +
            • +

              + paper on temperature, productivity referenced in book:Uninhabitable Earth +

              +
            • +
            • +

              + type:pdf habits digital interfaces +

              +
            • +
            • +

              + reading comprehension type:blog post +

              +
            • +
            • +

              + printer ink receipt +

              +
            • +
            • +

              + type:book read:2017 finance +

              +
            • +
            • +

              + file:py datetime parse +

              +
            • +
            +
          • +
          • +

            + Respect my privacy: hosted on something I control and never mined for ads. +

            +
          • +
          • +

            + Support all my devices with two-way sync so I can search and add to it wherever I am. +

            +
          • +
          • +

            + Be extensible: allow me to easily ingest my own information and extend with desired functionality. +

            +
          • +
          • +

            + Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of. +

            +
          • +
          • +

            + Notify me about changes to documents and webpages I’ve visited. +

            +
          • +
          • +

            + Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it. +

            +
          • +
          +

          + Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop +

          +

          + Parting Thoughts # +

          +

          + I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools. +

          +

          + It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share. +

          +

          + I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback. +

          +

          + HN Discussion +

          +

          + 2019-12-09: fixes grammar +

          +

          + Appendix # +

          +

          + The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS +

          +
          + +
          +
          + + + +
          + + +', 2285); + +COMMIT; diff --git a/packages/db/migrations/0140.undo.popular_read.sql b/packages/db/migrations/0140.undo.popular_read.sql new file mode 100755 index 000000000..c45addf8a --- /dev/null +++ b/packages/db/migrations/0140.undo.popular_read.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: popular_read +-- Description: Create omnivore.popular_read table + +BEGIN; + +DROP TABLE IF EXISTS omnivore.popular_read; + +COMMIT; From 3fa3526a166d8e62e9c601d978c0cfda2407fd06 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 15:16:02 +0800 Subject: [PATCH 072/115] fix wrong order of parameters in update scription api integration --- packages/rss-handler/src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 28a87bd3d..e3b8d896d 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -241,9 +241,9 @@ const getLink = (links: RssFeedItemLink[]) => { } const processSubscription = async ( - feedUrl: string, - userId: string, subscriptionId: string, + userId: string, + feedUrl: string, fetchResult: { content: string; checksum: string }, lastFetchedAt: number, scheduledAt: number, From 6a4d2a57f083c8709dda74cacf62e33a1ef43941 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 20 Oct 2023 17:54:20 +0800 Subject: [PATCH 073/115] First pass at label flair for iOS and Android --- android/Omnivore/app/build.gradle | 4 +- .../networking/SavedItemLabelMutations.kt | 6 +- .../omnivore/ui/components/LabelsViewModel.kt | 15 --- .../omnivore/ui/library/LibraryViewModel.kt | 75 ++++++------- .../ui/savedItemViews/SavedItemCard.kt | 105 ++++++++++++++++-- .../app/src/main/res/drawable/flair_feed.xml | 31 ++++++ .../main/res/drawable/flair_newsletter.xml | 16 +++ .../src/main/res/drawable/flair_pinned.xml | 13 +++ .../main/res/drawable/flair_recommended.xml | 16 +++ .../src/main/res/drawable/flaire_favorite.xml | 13 +++ apple/Omnivore.xcodeproj/project.pbxproj | 8 +- .../Views/FeedItem/LibraryItemCard.swift | 57 +++++++++- .../Sources/Views/Images/Images.swift | 7 ++ .../flair-favorite.imageset/Contents.json | 23 ++++ .../flair-favorite.imageset/Frame-2 1.png | Bin 0 -> 514 bytes .../flair-favorite.imageset/Frame-2 2.png | Bin 0 -> 739 bytes .../flair-favorite.imageset/Frame-2.png | Bin 0 -> 339 bytes .../flair-feed.imageset/Contents.json | 23 ++++ .../flair-feed.imageset/Frame 1.png | Bin 0 -> 491 bytes .../flair-feed.imageset/Frame 2.png | Bin 0 -> 513 bytes .../flair-feed.imageset/Frame.png | Bin 0 -> 397 bytes .../flair-newsletter.imageset/Contents.json | 23 ++++ .../flair-newsletter.imageset/Frame-1 1.png | Bin 0 -> 507 bytes .../flair-newsletter.imageset/Frame-1 2.png | Bin 0 -> 638 bytes .../flair-newsletter.imageset/Frame-1.png | Bin 0 -> 322 bytes .../flair-pinned.imageset/Contents.json | 23 ++++ .../flair-pinned.imageset/Frame-3 1.png | Bin 0 -> 473 bytes .../flair-pinned.imageset/Frame-3 2.png | Bin 0 -> 654 bytes .../flair-pinned.imageset/Frame-3.png | Bin 0 -> 351 bytes .../flair-recommended.imageset/Contents.json | 23 ++++ .../flair-recommended.imageset/Frame-4 1.png | Bin 0 -> 485 bytes .../flair-recommended.imageset/Frame-4 2.png | Bin 0 -> 621 bytes .../flair-recommended.imageset/Frame-4.png | Bin 0 -> 328 bytes 33 files changed, 401 insertions(+), 80 deletions(-) create mode 100644 android/Omnivore/app/src/main/res/drawable/flair_feed.xml create mode 100644 android/Omnivore/app/src/main/res/drawable/flair_newsletter.xml create mode 100644 android/Omnivore/app/src/main/res/drawable/flair_pinned.xml create mode 100644 android/Omnivore/app/src/main/res/drawable/flair_recommended.xml create mode 100644 android/Omnivore/app/src/main/res/drawable/flaire_favorite.xml create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Contents.json create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 1.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 2.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Contents.json create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 1.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 2.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Contents.json create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 1.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 2.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Contents.json create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 1.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 2.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Contents.json create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 1.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 2.png create mode 100644 apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4.png diff --git a/android/Omnivore/app/build.gradle b/android/Omnivore/app/build.gradle index e525d3c63..1041a5ec9 100644 --- a/android/Omnivore/app/build.gradle +++ b/android/Omnivore/app/build.gradle @@ -17,8 +17,8 @@ android { applicationId "app.omnivore.omnivore" minSdk 26 targetSdk 33 - versionCode 110 - versionName "0.0.110" + versionCode 118 + versionName "0.0.118" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" vectorDrawables { diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemLabelMutations.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemLabelMutations.kt index e7120536f..e4dd3561f 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemLabelMutations.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemLabelMutations.kt @@ -5,12 +5,12 @@ import app.omnivore.omnivore.graphql.generated.SetLabelsMutation import app.omnivore.omnivore.graphql.generated.type.CreateLabelInput import app.omnivore.omnivore.graphql.generated.type.SetLabelsInput -suspend fun Networker.updateLabelsForSavedItem(input: SetLabelsInput): Boolean { +suspend fun Networker.updateLabelsForSavedItem(input: SetLabelsInput): List? { return try { val result = authenticatedApolloClient().mutation(SetLabelsMutation(input)).execute() - return result.data?.setLabels?.onSetLabelsSuccess?.labels != null + return result.data?.setLabels?.onSetLabelsSuccess?.labels } catch (e: java.lang.Exception) { - false + return null } } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt index ddcdaf07b..c9dab6773 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt @@ -56,21 +56,6 @@ class LabelsViewModel @Inject constructor( serverSyncStatus = ServerSyncStatus.NEEDS_CREATION.rawValue ) - viewModelScope.launch { - withContext(Dispatchers.IO) { - dataService.db.savedItemLabelDao().insertAll(listOf(res)) - - val newLabel = networker.createNewLabel(CreateLabelInput(color = presentIfNotNull(res.color), name = res.name)) - if (newLabel != null) { - try { - dataService.db.savedItemLabelDao().updateTempLabel(tempId, newLabel.id) - } catch (e: Exception) { - Log.d("EXCEPTION: ", e.toString()) - } - } - } - } - return res } } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt index 632b32950..86dfab019 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt @@ -304,55 +304,44 @@ class LibraryViewModel @Inject constructor( fun updateSavedItemLabels(savedItemID: String, labels: List) { viewModelScope.launch { withContext(Dispatchers.IO) { - val syncedLabels = labels.filter { it.serverSyncStatus == ServerSyncStatus.IS_SYNCED.rawValue } - val unsyncedLabels = labels.filter { it.serverSyncStatus != ServerSyncStatus.IS_SYNCED.rawValue } + val input = SetLabelsInput( + pageId = savedItemID, + labels = Optional.presentIfNotNull(labels.map { CreateLabelInput(color = Optional.presentIfNotNull(it.color), name = it.name) }), + ) - var labelCreationError = false - val createdLabels = unsyncedLabels.mapNotNull { label -> - val result = networker.createNewLabel(CreateLabelInput( - name = label.name, - color = presentIfNotNull(label.color), - description = presentIfNotNull(label.labelDescription), - )) - result?.let { + val updatedLabels = networker.updateLabelsForSavedItem(input) + + // Figure out which of the labels are new + updatedLabels?.let { updatedLabels -> + val existingNamedLabels = dataService.db.savedItemLabelDao() + .namedLabels(updatedLabels.map { it.labelFields.name }) + val existingNames = existingNamedLabels.map { it.name } + val newNamedLabels = updatedLabels.filter { !existingNames.contains(it.labelFields.name) } + + dataService.db.savedItemLabelDao().insertAll(newNamedLabels.map { SavedItemLabel( - savedItemLabelId = result.id, - name = result.name, - color = result.color, - createdAt = result.createdAt.toString(), - labelDescription = result.description, - serverSyncStatus = ServerSyncStatus.IS_SYNCED.rawValue + savedItemLabelId = it.labelFields.id, + name = it.labelFields.name, + color = it.labelFields.color, + createdAt = null, + labelDescription = null + ) + }) + + val allNamedLabels = dataService.db.savedItemLabelDao() + .namedLabels(updatedLabels.map { it.labelFields.name }) + val crossRefs = allNamedLabels.map { + SavedItemAndSavedItemLabelCrossRef( + savedItemLabelId = it.savedItemLabelId, + savedItemId = savedItemID ) - } ?: run { - labelCreationError = true - null } - } + dataService.db.savedItemAndSavedItemLabelCrossRefDao().deleteRefsBySavedItemId(savedItemID) + dataService.db.savedItemAndSavedItemLabelCrossRefDao().insertAll(crossRefs) - dataService.db.savedItemLabelDao().insertAll(createdLabels) - - val allLabels = syncedLabels + createdLabels - - val input = SetLabelsInput(labelIds = Optional.presentIfNotNull(allLabels.map { it.savedItemLabelId }), pageId = savedItemID) - val networkResult = networker.updateLabelsForSavedItem(input) - - val crossRefs = allLabels.map { - SavedItemAndSavedItemLabelCrossRef( - savedItemLabelId = it.savedItemLabelId, - savedItemId = savedItemID - ) - } - - // Remove all labels first - dataService.db.savedItemAndSavedItemLabelCrossRefDao().deleteRefsBySavedItemId(savedItemID) - - // Add back the current labels - dataService.db.savedItemAndSavedItemLabelCrossRefDao().insertAll(crossRefs) - - if (!networkResult || labelCreationError) { - snackbarMessage = resourceProvider.getString(R.string.library_view_model_snackbar_error) - } else { snackbarMessage = resourceProvider.getString(R.string.library_view_model_snackbar_success) + } ?: run { + snackbarMessage = resourceProvider.getString(R.string.library_view_model_snackbar_error) } CoroutineScope(Dispatchers.Main).launch { diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/savedItemViews/SavedItemCard.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/savedItemViews/SavedItemCard.kt index 654fcd22c..be0de7ba5 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/savedItemViews/SavedItemCard.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/savedItemViews/SavedItemCard.kt @@ -20,16 +20,20 @@ import androidx.compose.ui.focus.onFocusChanged import androidx.compose.ui.graphics.Color import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.colorResource +import androidx.compose.ui.res.painterResource import androidx.compose.ui.text.TextStyle import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.intl.Locale import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.text.toLowerCase +import androidx.compose.ui.text.toUpperCase import androidx.compose.ui.unit.* import app.omnivore.omnivore.R +import app.omnivore.omnivore.persistence.entities.SavedItemLabel import app.omnivore.omnivore.persistence.entities.SavedItemWithLabelsAndHighlights import app.omnivore.omnivore.ui.components.LabelChipColors import app.omnivore.omnivore.ui.library.SavedItemAction +import app.omnivore.omnivore.ui.library.SavedItemFilter import app.omnivore.omnivore.ui.library.SavedItemViewModel import coil.compose.rememberAsyncImagePainter @@ -45,14 +49,14 @@ fun SavedItemCard( Column( modifier = Modifier - .combinedClickable( - onClick = onClickHandler, - onLongClick = { - savedItemViewModel.actionsMenuItemLiveData.postValue(savedItem) - } - ) - .background(if (selected) MaterialTheme.colorScheme.surfaceVariant else MaterialTheme.colorScheme.background) - .fillMaxWidth() + .combinedClickable( + onClick = onClickHandler, + onLongClick = { + savedItemViewModel.actionsMenuItemLiveData.postValue(savedItem) + } + ) + .background(if (selected) MaterialTheme.colorScheme.surfaceVariant else MaterialTheme.colorScheme.background) + .fillMaxWidth() ) { Row( horizontalArrangement = Arrangement.SpaceBetween, @@ -108,8 +112,10 @@ fun SavedItemCard( ) } - FlowRow(modifier = Modifier.fillMaxWidth().padding(10.dp)) { - savedItem.labels.sortedWith(compareBy { it.name.toLowerCase(Locale.current) }).forEach { label -> + FlowRow(modifier = Modifier + .fillMaxWidth() + .padding(10.dp)) { + savedItem.labels.filter { !isFlairLabel(it) }.sortedWith(compareBy { it.name.toLowerCase(Locale.current) }).forEach { label -> val chipColors = LabelChipColors.fromHex(label.color) LabelChip( @@ -196,6 +202,82 @@ fun readingProgress(item: SavedItemWithLabelsAndHighlights): String { // return "" //} + +public enum class FlairIcon( + public val rawValue: String, + public val sortOrder: Int +) { + FEED("feed", 0), + RSS("rss", 0), + FAVORITE("favorite", 1), + NEWSLETTER("newsletter", 2), + RECOMMENDED("recommended", 3), + PINNED("pinned", 4) +} + +val FLAIR_ICON_NAMES = listOf("feed", "rss", "favorite", "newsletter", "recommended", "pinned") + +fun isFlairLabel(label: SavedItemLabel): Boolean { + return FLAIR_ICON_NAMES.contains(label.name.toLowerCase(Locale.current)) +} + +@Composable +fun flairIcons(item: SavedItemWithLabelsAndHighlights) { + val labels = item.labels.filter { isFlairLabel(it) }.map { + FlairIcon.valueOf(it.name.toUpperCase(Locale.current)) + } + labels.forEach { + when (it) { + FlairIcon.RSS, + FlairIcon.FEED -> { + Image( + painter = painterResource(id = R.drawable.flair_feed), + contentDescription = "Feed flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + FlairIcon.FAVORITE -> { + Image( + painter = painterResource(id = R.drawable.flaire_favorite), + contentDescription = "Favorite flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + FlairIcon.NEWSLETTER -> { + Image( + painter = painterResource(id = R.drawable.flair_newsletter), + contentDescription = "Newsletter flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + FlairIcon.RECOMMENDED -> { + Image( + painter = painterResource(id = R.drawable.flair_recommended), + contentDescription = "Recommended flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + FlairIcon.PINNED -> { + Image( + painter = painterResource(id = R.drawable.flair_pinned), + contentDescription = "Pinned flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + } + } +} + @Composable fun readInfo(item: SavedItemWithLabelsAndHighlights) { Row( @@ -203,6 +285,9 @@ fun readInfo(item: SavedItemWithLabelsAndHighlights) { .fillMaxWidth() .defaultMinSize(minHeight = 15.dp) ) { + // Show flair here + flairIcons(item) + Text( text = estimatedReadingTime(item), style = TextStyle( diff --git a/android/Omnivore/app/src/main/res/drawable/flair_feed.xml b/android/Omnivore/app/src/main/res/drawable/flair_feed.xml new file mode 100644 index 000000000..aca102b59 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flair_feed.xml @@ -0,0 +1,31 @@ + + + + + + + + diff --git a/android/Omnivore/app/src/main/res/drawable/flair_newsletter.xml b/android/Omnivore/app/src/main/res/drawable/flair_newsletter.xml new file mode 100644 index 000000000..5c70af531 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flair_newsletter.xml @@ -0,0 +1,16 @@ + + + + + + + diff --git a/android/Omnivore/app/src/main/res/drawable/flair_pinned.xml b/android/Omnivore/app/src/main/res/drawable/flair_pinned.xml new file mode 100644 index 000000000..25b42df77 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flair_pinned.xml @@ -0,0 +1,13 @@ + + + + + + diff --git a/android/Omnivore/app/src/main/res/drawable/flair_recommended.xml b/android/Omnivore/app/src/main/res/drawable/flair_recommended.xml new file mode 100644 index 000000000..88a5ea8d3 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flair_recommended.xml @@ -0,0 +1,16 @@ + + + + + + + diff --git a/android/Omnivore/app/src/main/res/drawable/flaire_favorite.xml b/android/Omnivore/app/src/main/res/drawable/flaire_favorite.xml new file mode 100644 index 000000000..820704635 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flaire_favorite.xml @@ -0,0 +1,13 @@ + + + + + + diff --git a/apple/Omnivore.xcodeproj/project.pbxproj b/apple/Omnivore.xcodeproj/project.pbxproj index 32ee2a7a5..805222b68 100644 --- a/apple/Omnivore.xcodeproj/project.pbxproj +++ b/apple/Omnivore.xcodeproj/project.pbxproj @@ -1400,7 +1400,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.34.0; + MARKETING_VERSION = 1.35.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; @@ -1435,7 +1435,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.34.0; + MARKETING_VERSION = 1.35.0; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1490,7 +1490,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.34.0; + MARKETING_VERSION = 1.35.0; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = Omnivore; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -1831,7 +1831,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.34.0; + MARKETING_VERSION = 1.35.0; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = Omnivore; PROVISIONING_PROFILE_SPECIFIER = ""; diff --git a/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift b/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift index fff8f9b71..83350cd70 100644 --- a/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift +++ b/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift @@ -2,6 +2,35 @@ import Models import SwiftUI import Utils +enum FlairLabels: String { + case pinned + case favorite + case recommended + case newsletter + case rss + case feed + + var icon: Image { + switch self { + case .pinned: return Image.flairPinned + case .favorite: return Image.flairFavorite + case .recommended: return Image.flairRecommended + case .newsletter: return Image.flairNewsletter + case .feed, .rss: return Image.flairFeed + } + } + + var sortOrder: Int { + switch self { + case .feed, .rss: return 0 + case .favorite: return 1 + case .newsletter: return 2 + case .recommended: return 3 + case .pinned: return 4 + } + } +} + public extension View { func draggableItem(item: LinkedItem) -> some View { #if os(iOS) @@ -124,8 +153,30 @@ public struct LibraryItemCard: View { return "" } + var flairLabels: [FlairLabels] { + item.sortedLabels.compactMap { label in + if let name = label.name { + return FlairLabels(rawValue: name.lowercased()) + } + return nil + }.sorted { $0.sortOrder < $1.sortOrder } + } + + var nonFlairLabels: [LinkedItemLabel] { + item.sortedLabels.filter { label in + if let name = label.name, FlairLabels(rawValue: name.lowercased()) != nil { + return false + } + return true + } + } + var readInfo: some View { - AnyView(HStack { + HStack(alignment: .center, spacing: 5.0) { + ForEach(flairLabels, id: \.self) { + $0.icon + } + let fgcolor = Color.isDarkMode ? Color.themeDarkWhiteGray : Color.themeMiddleGray Text("\(estimatedReadingTime)") .font(.caption2).fontWeight(.medium) @@ -146,7 +197,7 @@ public struct LibraryItemCard: View { .font(.caption2).fontWeight(.medium) .foregroundColor(fgcolor) } - .frame(maxWidth: .infinity, alignment: .leading)) + .frame(maxWidth: .infinity, alignment: .leading) } var imageBox: some View { @@ -227,6 +278,6 @@ public struct LibraryItemCard: View { } var labels: some View { - LabelsFlowLayout(labels: item.sortedLabels) + LabelsFlowLayout(labels: nonFlairLabels) } } diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.swift b/apple/OmnivoreKit/Sources/Views/Images/Images.swift index d32ef814d..ce00165cb 100644 --- a/apple/OmnivoreKit/Sources/Views/Images/Images.swift +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.swift @@ -28,4 +28,11 @@ public extension Image { static var unarchive: Image { Image("unarchive", bundle: .module) } static var remove: Image { Image("remove", bundle: .module) } static var label: Image { Image("label", bundle: .module) } + + static var flairFeed: Image { Image("flair-feed", bundle: .module) } + static var flairFavorite: Image { Image("flair-favorite", bundle: .module) } + + static var flairNewsletter: Image { Image("flair-newsletter", bundle: .module) } + static var flairPinned: Image { Image("flair-pinned", bundle: .module) } + static var flairRecommended: Image { Image("flair-recommended", bundle: .module) } } diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Contents.json new file mode 100644 index 000000000..6738e3537 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "filename" : "Frame-2.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "filename" : "Frame-2 1.png", + "idiom" : "universal", + "scale" : "2x" + }, + { + "filename" : "Frame-2 2.png", + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 1.png new file mode 100644 index 0000000000000000000000000000000000000000..84b84ba31e2bcf7e22ff199ab545033b2653ff3b GIT binary patch literal 514 zcmV+d0{#7oP)L zHwH@Qb%vF0dwW`;dizMuM6u!uME{TSD&vnp*G35WO@7&R-IO;i4=O2|2au$QTvn z;zWG9%ViM>L8##XsAyb8#*Ng;$z<0)I;EDxx-_w`mWoLP$v3rO^To_nJxCCYDGel13o4 zL=us-0$WL#M79ywiIJ$NU$wGaJo+8SU?Flefiow|H)4ySXp|4ep1j&#<_nwfk8hM*w7sXZP z?|(gpslxc*N@NV*kE_f#sU@-X-%bS441`I+u$TlbFnST03wko27k>u(f;LLfO^~ja zs7NJ=?df?a_TFvD;!O@RyqLE0%LbN89KJn8cgXqX%8~S5 z!{}n{r{V{mnKBWX_kdGC3*5m02x7jIZ?5y6=Ss-EH$%e|=mP;c(?`mh(pHH_okm2M z!XZeX$(>BByJ3|eC9dEg5_DzT4ZB3WtMF(XcEeA`W;`RosRW14f{9Rv>-s-35-G_> zzX!)cU$?K#SVlsUa9VHP3r4izc6>Uq9(r#Ag-xBk*lnG^W}+ws$cVByMqJ zst7QZyB)_CsVczKYuOx0q>cbnyLr~PNL>M@Q3tRuktPC6BMGK$Kgy)30Ml54`P-NJ zf|ep}1ejJ5Ok0t*0!(WOrm0950*pilyxOlfiAV2HtJV@XT?jCpNHBFoxR^Utt7~*b zC+YN*veY_MqXorW6FQe*@nBF9qEzUzof2!Mdah<=f1_lNO@EhVe V8A8u6pN;?k002ovPDHLkV1juFImZA1 literal 0 HcmV?d00001 diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2.png new file mode 100644 index 0000000000000000000000000000000000000000..5e93251c38e85f3b74e0078124c4ded44530a95b GIT binary patch literal 339 zcmeAS@N?(olHy`uVBq!ia0vp^f*{Pn1|+R>-G2co&H|6fVg?3oVGw3ym^DWND9BhG z%Mf#JQXVcY}`sDwot-1%CEbT&icP6uGy4iSB{*ET{QXZn%~7%POQX zdGS5&T<>ltu6$a}^YfZSBfFIw!(7gYsU5pzseATPHJ^u~fYnvCqUA;(q&$vm?Wo-;U-|WzR#WQh&gn`MIc~hS1JT@@t*wb`o-eQ3(N_@WrE7vG+ z{Oe8G^ixF1VSU7vJ^Mu%Uz%+`W@gSvXvq1MvK48l9zwiJbV)~^d$ zckG)7j>!#Vbnv*HDvTt406~y6A195>;PZVr)lk|XI6PW)hl`9Y+(^Py#ucuV;2TdY zNUyZ6wt5*B+b~5cIHzHV&yNdu9)xtJmEd#{q5Rsnrv(_9Rz?e_OR-MaOb%?@k~iTe z;>xk5v@(8#pvtvyqFfvYYe`+0hBB^jBMDO(SGbjgQU>${;=~2+B%zRTf{&8;CF9|| h?u;?N{B!2?^9|twC(QWd%0&PG002ovPDHLkV1k{0&9wjk literal 0 HcmV?d00001 diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 2.png new file mode 100644 index 0000000000000000000000000000000000000000..358758a7e259206b6e4539ce2372d597dd501c9a GIT binary patch literal 513 zcmV+c0{;DpP)W$O$Nli|p0D>S0f-sIkA@say@cA)CrDo7rLaAGI_uZlZ>=I7+#u>LIWY{DE;ayl| zvL%S&U07t2B{IXiWXX^vn8Ujy$wW&O4(~#e2}=BPc$X*{uY|s;Pf_D(1L0ks)uOu? z8&>d9VY&qZE6(-x-r?BLF9H3AZ9u73-G42|#$E}v=pWH_2WImHo4|xOuZcM)Cf?l; ziN3@>>UwTA&rq6Gc!2NfJBCIlr;q7u{k|mQF-X6v`9P~u+et;ws5#A}; zS6@%~C2s0>^L8@$sLhh0hyUv+T^)^GW&;w{%v7X)U2@2djL4p7IL}pKJsU_{bNKs2UFp0r9F@P0XEm4pXD4+>Kc zd?*X;9T}{Pg{s;cn}}f9icVK~#7F?UvtB z!axwlzs=ygC4>^J0IYx##0nVSG%`anqrpcz06NeLqytMZCFEJgtb2xJE^zrv;_#5) zOfttFx!wJCFFBy6r^gH#t#clOF_F? zOGws_{b(i=4!d$+pk3!7B6wKR>+w_`_iT$!A2PCWdH`Ge;1(|wn5MJknDauUvj4{F z0iT94^3dFQC=GsxXSz-UQ0vD^=L8W+Lw-y44o_2?Pp;TiFahcXpt8Tp$Uo;9cKDRJ zk^a|#;c;886&`tn3sJ$94B{+?m3k$qy{|b}+3#e{Xy5b)O{%AWIg-2lbd zsG>@h>z*;7pk9g`}Gn%V1=^+X;v zl3t1Aqhw)LrS@$H5u;Qgxfmx=zZ@OV2a%8sf@*#rItc>O>Hh&^qyIcUPdxBitT*T~ xfF(^)7=$8z5lv*a&4X?r>upR2-JYJa19osN?_u002ovPDHLkV1mOk+5i9m literal 0 HcmV?d00001 diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 2.png new file mode 100644 index 0000000000000000000000000000000000000000..b6d39f40c17f53d46b88f7c1c91833ececa15514 GIT binary patch literal 638 zcmV-^0)hRBP)vl5z6Q zwBt0PlmFjMSQj7=2m}Iwe~u`e5y>XLaZNLukaSf`OxGMEG0&TnFJd5vG^2ll#qT08 zLjyfTBIbup4CI9FUHHS2t_~NK8aq9bExNbi7b>0G+<-v@g>6l$C8qF$m{!`sSiEwB z#T$DdyL2DY)xv3b!#N@{c4?YWEJATd=_Tkz0=K9R<$e#3VpcRUn@;LTP4i6YH5Og^ ztB2&+bB%KUiWQLT*B!DVesD=CwBRPw$?)!YuOzuY68%j2D0A=S3s#t75nF6V`N0z$`I2cJ8Y?}T;@s!^?XI;OxeY& zviwPO^hd`;LdP5xPJjjXb!>^)V%{s8(c^O8gl!QDJ}Q>o*QvuD(O(%qr98eM_q%X} zOA(3|u-Uv$agk1y6GZqz5aA115p^u#GcsEtbCHPYT=!YXXZiut?VwsmQq}1v@ut*I z!S9KYP(7yHPIToY*>S@SnKyRBvo@yWo5_me4xsA(ozd?Tr)uxl=>La3x`2T|AP{iE YFO<>wJ_@8-G2co&H|6fVg?3oVGw3ym^DWND9BhG zk=-oP z7U9H|df2gaf#E`4D@UIFJav*)JEtCVS5k2dc3Z`|@i@oB&;3l35*xRA8ZMj9!F|hW zl4$;UjtM)I(rPlCxL8}3wjAwS_v)48xf;JO_dJ6?EnU4~)w;r#y_FG*r5s}yc P^df_&tDnm{r-UW|2nKmo literal 0 HcmV?d00001 diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Contents.json new file mode 100644 index 000000000..a55c67c87 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "filename" : "Frame-3.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "filename" : "Frame-3 1.png", + "idiom" : "universal", + "scale" : "2x" + }, + { + "filename" : "Frame-3 2.png", + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 1.png new file mode 100644 index 0000000000000000000000000000000000000000..dba4109e82456de23d833984f77e85ee1aa0a320 GIT binary patch literal 473 zcmV;~0Ve*5P)pmc(cP<4W?kZ#Zo;s?ZQvq2ew1%Q4hH4-AmzOx-E4}21` zF>v?8_Xm{PwCMqrsYI%(YBvnS52hy1^H*a`&wW;_L>kAj#!%pR#{3MZg8OU|2-*b< zFW2*Fnhp|>=x`7KIfUcgjqxnIXswUF&_qGF03;y65F7w5h!}$64Cil{)*Jv12n*sr z!44?FcRE#pS_AP`M_HES{1?YL27p9W13UF{UHbsgc*r@s<~{0R?|Zqp8je{9Oaair zIfbFKU0mIXSqHo;fD-}dpWpXLU$=K`4M1A_fEidkz4}TcTu$YFii@!v$bL z1nNt2vlk*H9JXE%3}FH2F!zP#B>AgyLf8N(+nc5_jvS$m_@RW#Bmnbdv)t38!yenV zy(poqI{{;#E&$S^YJvesPb=$%3t$d{VXq=(g`i({ycvB#jWo$NZAzg(dZPq=_b~S; P00000NkvXXu0mjfGgP>= literal 0 HcmV?d00001 diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 2.png new file mode 100644 index 0000000000000000000000000000000000000000..487a9b2730a3473778309fe8a3ec9b5fcdeb3d00 GIT binary patch literal 654 zcmV;90&)F`P)JT zf#23CJxG8Xx65C=WZG&KqgsRART=Wtab%5OEex97>HwV}@e0?66K~IgvQ^h530X zl~q#}u`VGfP*hhDu__@cP@EO}z;qpqB8eYO+xm=FR4oGM0Q-}TL zOiS(r{%&1$KuSV$#L=a4#w#37w-CU+CP5Hn*ro`jX;nQTi5m%wR;$%eq^@WN`i!{~%GP+DUf7D1OwQqn;ALjj9m(XonJGWERU~Ji?+c+1XR%g@guVh5%C+PLZkr!=VLv`_Tv*0~;!$K$!=7u$}n&5w{9KmY&$07*qoM6N<$f>HPk1ONa4 literal 0 HcmV?d00001 diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3.png new file mode 100644 index 0000000000000000000000000000000000000000..1d0df2685ef61bdf9ccb7dd654b25cbdbb43f6d3 GIT binary patch literal 351 zcmeAS@N?(olHy`uVBq!ia0vp^f*{Pn1|+R>-G2co&H|6fVg?3oVGw3ym^DWND9BhG zg^*POcuT43k@u{9d*-iyzju904{M3Xr1!r! zzTVZpNMnaq9-pEiPjYnV)vKTP+>BRw@hIT>R*h{MOT|SeF4X2beBsb*>lxabM;`WX z(mB0VC8;hZ`=Ile2QN+~_UtV)$#id0h*#J(+ec2@FX}!lCkF0|}OD=R~B%ET|kz{Bd7rTdh;|ZB8 ul_%0ITP(_d8Xw#)Sl`BT)c@q-Bf{%m$7r(MIldq0e+ExiKbLh*2~7YC;E9C* literal 0 HcmV?d00001 diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Contents.json new file mode 100644 index 000000000..239db47c1 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "filename" : "Frame-4.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "filename" : "Frame-4 1.png", + "idiom" : "universal", + "scale" : "2x" + }, + { + "filename" : "Frame-4 2.png", + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 1.png new file mode 100644 index 0000000000000000000000000000000000000000..a1ffdd0884641d8567c9dc4197f446432e70d463 GIT binary patch literal 485 zcmVt}>+Y?)PB2GY7@lAZLh(dgzzn=5<uFQ6v``IH+1gk^d}DFz7RUAU_{xpqVtGJc4}H_Pg`E?Uq%GDC(GwX^;P b)EqZoB_(+jD&f8?00000NkvXXu0mjf^wh#t literal 0 HcmV?d00001 diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 2.png new file mode 100644 index 0000000000000000000000000000000000000000..3892567f1dbcd8d4f160cd3b00fb89d8d82fdf5c GIT binary patch literal 621 zcmV-z0+RiSP)K-{Wxqb%c!Y-6HOBC3`oCzwycd|79{X@)|U=lVX@k^M= zYg&RxG!=N_E3^3@3&5G3?8K$XoEX0aIKd`?ZmP>~eXPJsBT(XZRAz=h|GL8*rv93-&z8g`dMj{4YX}9xsV$H|s5U|jM{gII7V&3yeI&agKgsp

          #wAQg$) zHG3x_LC*M*goJBQTau7)4eF*HDc~5?mJ}rFzDS;c1of^dmR_Q7P+OK>LagAVw@=5O z8cPBb&V1fsv?SgKk;3AsgrFmzj-9uYUDw(l>1s0%4!$kw%_7Q5hpyh(oWr*7n-G2co&H|6fVg?3oVGw3ym^DWND9BhG zKCM6@bWb6T66D!@(xXot&+jwTU(b+DZ9+?nRep&*%SqbaF@{OGFh3* zEIt)qm#4k|{8d{~uzg*ZQrX8+yG17}W1k7f9q`pIU!k*o%a_W%0udfprpO=K%^sL&z?oRQE4ly-gPYR%44}b{(&p|V;Su-BU?-^ zKCu6COkmcmYHyC2HP<)}^|)tu2;6x7`OW(euI$&GIu*?-oNw_r1u-OKr14m-)L+JL W|M0;|Qx>2{89ZJ6T-G@yGywp-JAMlQ literal 0 HcmV?d00001 From 3bd43048a4dd50afe4da38e3a9bfb2d0b7f54a71 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 15:35:29 +0800 Subject: [PATCH 074/115] add test case for forte labs newsletter --- .../forte-labs/expected-metadata.json | 12 + .../newsletters/forte-labs/expected.html | 19 + .../newsletters/forte-labs/source.html | 1736 +++++++++++++++++ .../test-pages/newsletters/forte-labs/url.txt | 1 + 4 files changed, 1768 insertions(+) create mode 100644 packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected-metadata.json create mode 100644 packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html create mode 100644 packages/readabilityjs/test/test-pages/newsletters/forte-labs/source.html create mode 100644 packages/readabilityjs/test/test-pages/newsletters/forte-labs/url.txt diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected-metadata.json b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected-metadata.json new file mode 100644 index 000000000..f15f6cdab --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "Progressive Summarization: A Practical Technique for Designing Discoverable Notes", + "byline": "Tiago Forte", + "dir": null, + "excerpt": "Modern digital tools make it easy to “capture” information from a wide variety of sources. We know how to snap a picture, type out some notes, record a video, or scan a document. Getting this content from the outside world into the digital world is trivial. It’s even easier to get content that is already digital from one app to another. We know how to copy and paste text, save an image from a webpage, archive an email attachment, or import a video file.", + "siteName": "Forte Labs", + "siteIcon": "https://fortelabs.com/wp-content/uploads/2020/02/cropped-cropped-Icon_Red-1-32x32.png", + "previewImage": "https://fortelabs.com/wp-content/uploads/2017/12/1vTBT177SuhcgBEdPJwT9lA.jpeg", + "publishedDate": "2017-12-27T14:59:00.000Z", + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html new file mode 100644 index 000000000..67dae7ff0 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html @@ -0,0 +1,19 @@ +

          + + + + +

          word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word


          + mmMwWLliI0fiflO&1
          + mmMwWLliI0fiflO&1
          + mmMwWLliI0fiflO&1
          + mmMwWLliI0fiflO&1
          + mmMwWLliI0fiflO&1
          + mmMwWLliI0fiflO&1
          + mmMwWLliI0fiflO&1 +
          \ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/source.html b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/source.html new file mode 100644 index 000000000..44a7a65d5 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/source.html @@ -0,0 +1,1736 @@ + + + + + + + + + + + + Progressive Summarization: A Practical Technique for Designing Discoverable Notes - Forte Labs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          + + + +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +

          + Progressive Summarization: A Practical Technique for Designing Discoverable Notes +

          +
          +
          + + +
          +
          +

          + Estimated reading time: 12 minutes +

          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + Series Navigation: Progressive Summarization +
          + + + + + + + +
          + Progressive Summarization II: Examples and Metaphors >> +
          +
          +

          +

          + +

          +

          + Modern digital tools make it easy to “capture” information from a wide variety of sources. We know how to snap a picture, type out some notes, record a video, or scan a document. Getting this content from the outside world into the digital world is trivial. +

          +

          + It’s even easier to get content that is already digital from one app to another. We know how to copy and paste text, save an image from a webpage, archive an email attachment, or import a video file. +

          +

          + What is difficult is not transferring content from place to place, but transferring it through time. +

          +

          + You know what I mean: you read a book, investing hours of mental labor in understanding the ideas it presents. You finish the book with a feeling of triumph that you’ve gained a valuable body of knowledge. +

          +

          + But then what? +

          +

          + You may try to apply the science-based methods the book recommends, only to realize it’s not quite as clear-cut as you thought. You may try to change the way you eat, exercise, communicate, or work, trusting in the power of habits. But then the everyday demands of life come rushing back, and you forget what motivated you in the first place. +

          +

          + At this point, people take different paths. Some give up, labeling all “self-help” books a waste of time. Others decide it’s just a problem of remembering everything they read, and invest in fancy memorization techniques. And many people become “infovores,” force-feeding themselves endless books, articles, and courses, in the hope that something will stick. +

          +

          + I want to suggest an alternative to all the approaches above: what you read is good and useful and very important, you’re just reading it at the wrong time. +

          +

          + You’re reading about time management techniques now, but they will only be useful two years from now, when you become a manager and have much greater demands on your time. +

          +

          + You’re watching YouTube videos on online marketing now, but that knowledge can only be put to use in 9 months, when your new online course gets off the ground. +

          +

          + You’re talking to a prospect about his goals and challenges now, but when you could really use that information is next year, when he is taking bids for a huge new contract. +

          +

          + The challenge of knowledge is not acquiring it. In our digital world, you can acquire almost any knowledge at almost any time. +

          +

          + The challenge is knowing which knowledge is worth acquiring. And then building a system to forward bits of it through time, to the future situation or problem or challenge where it is most applicable, and most needed. +

          +

          + At that future point, when you’re applying that knowledge directly to a real-world challenge, you won’t have to worry about memorizing it, integrating it, or even fully understanding it. You will only have to apply it, and any gaps in your understanding will very quickly reveal themselves. By the time you’re done solving a real problem with it, book knowledge has become experiential knowledge. And experiential knowledge is something you carry with you forever. +

          +

          + This is the job of a “second brain” — an external, integrated digital repository for the things you learn and the resources from which they come. It is a storage and retrieval system, packaging bits of knowledge into discrete packets that can be forwarded to various points in time to be reviewed, utilized, or deleted. +

          +
          +
          +
          +
          +

          + Use Progressive Summarization
          + to create easy-to-review notes +

          +
          +
          +

          + I'll send you my Progressive Summarization Cheat Sheet as a thank you when you subscribe to my free weekly newsletter below. +

          +
          +
            +
            +
            + +
            +
            +
            +
            +

            + Look out for an email from hello@fortelabs.co +

            +
            +
            +
            +

            +   +

            +

            + In The PARA Method, I described a universal system for organizing any kind of digital information from any source. It is a “good enough” system, maintaining notes according to their actionability (which takes just a moment to determine), instead of their meaning (which is ambiguous and depends on the context). +

            +

            + The four top-level categories of PARA — Projects, Areas, Resources, and Archives — are designed to facilitate this process of forwarding knowledge through time. +

            +
              +
            • By placing a note in a project folder, you are essentially scheduling it for review on the short time horizon of an individual project +
            • +
            • Notes in area folders are scheduled for less frequent review, whenever you evaluate that area of your work or life +
            • +
            • Notes in resource folders stand ready for review if and when you decide to take action on that topic +
            • +
            • And notes in archive folders are in “cold storage,” available if needed but not scheduled for review at any particular time +
            • +
            +

            + Note that we have re-created the tickler file, except instead of strict time-based horizons (daily, weekly, monthly, annually), they are scheduled contingently — if X happens, when Y arrives, if I want to do Z, etc. +

            +

            + +

            +

            + Planning in terms of contingencies gives us all the benefits of planning and researching, without locking us into rigid routines. We have the ability to massively accelerate, using our repository of accumulated notes as rocket fuel. But the actual decision of whether or not to accelerate, and critically, in which direction, we leave to our Future Self, who is older and wiser. +

            +

            + PARA answers how these “packets of knowledge” are organized: in discrete notes, sorted into 4 categories according to actionability, and resurfaced using RandomNote. +

            +

            + But now we turn to a more fundamental question: how are these packets made? Once we capture something, how do we structure the note so that it’s easily discoverable and usable in the future? How do we make sure what we’re saving today adds value to future projects, even when we can’t predict or even imagine what those projects might be? +

            +

            + That is the job of Progressive Summarization. +

            +

            + Note-first knowledge management +

            +

            + There are two primary schools of thought on how to organize a note-taking program (or really any body of information, but I’ll use terms specific to note-taking apps): +

            +

            + +

            +

            + Tagging-first approaches argue that there should be no explicit hierarchy of notes, notebooks, and stacks. Notes are envisioned as an ever-changing, virtual matrix of interconnected, free-floating ideas. Because many tags can be applied to one note, there are multiple pathways to discover any given note. Locating notes in specific notebooks and folders is seen as limiting and static. +

            +

            + +

            +

            + Although tags have their uses, I don’t believe they work as a primary organizational system. In my experience, relying on tagging is too fragile and requires too much maintenance, spreading attention too uniformly across all notes whether or not they are truly valuable. The virtual matrix sounds cool and futuristic, but our minds are not made to work well with such abstract concepts — we understand placing one thing in one place intuitively and automatically. +

            +

            + The second conventional approach to organizing notes is notebook-first. This basically translates how we organize things in the physical world — in a series of discrete containers — into the digital world. +

            +

            + +

            +

            + Notebook-first is better than tagging-first, in my opinion, mostly because it stays out of the way. It doesn’t try to automate and encroach upon the deeply intuitive act of making connections and seeing patterns. PARA on its own is a notebook-first system. +

            +

            + But if we stopped there, it would still be woefully inadequate for an economy based on creative output. As the tagging enthusiasts correctly point out, notebooks and folders actually suppress the serendipity and randomness that is at the heart of a creative lifestyle. +

            +

            + I propose a way to break the impasse: a note-first approach. +

            +

            + +

            +

            + I propose we make the design of individual notes the primary factor, instead of tags or notebooks. This has many advantages: +

            +
              +
            • It works well with any other organizational system, without depending on them (including but not limited to tags and notebooks, if you want to use those) +
            • +
            • It makes all work you do on your notes value-added, because you’re spending close to 100% of the time engaging directly with the content itself +
            • +
            • It can more easily survive migrations to other devices, storage locations, and even programs, because note content is much more likely to be preserved than overarching structure +
            • +
            • It cultivates skills (succinct communication, finding the core of an idea, visual thinking, etc.) that are inherently valuable and highly transferrable to other activities +
            • +
            • It makes your notes more legible and useful to others (unlike your internal notebook structure, which is only for your use), promoting collaboration and sharing +
            • +
            +

            + With a note-first approach, your notes become like individual atoms — each with its own unique properties, but ready to be assembled into elements, molecules, and compounds that are far more powerful. +

            +

            + Designing discoverable notes +

            +

            + A note-first approach to knowledge management means we have to think about design. You are, in a very real sense, designing a product for a demanding customer — Future You. +

            +

            + Future You doesn’t necessarily trust that everything Past You put into your notes is valuable. Future You is impatient and skeptical, demanding proof upfront that the time they spend reviewing notes will be worthwhile. You’ve gotta “sell them” on the idea of reviewing a given note, including all the stages any salesperson has to master: gaining attention, inspiring interest, establishing credibility, stoking desire, and making a case for action NOW. +

            +

            + As if all that wasn’t intimidating enough, you have to do this for every single note without spending any extra time. You don’t have extra time, do you? +

            +

            + Let’s start at the beginning: at the heart of every design, we are trying to balance priorities. You want one thing, but it has to be balanced against something else that you also want. +

            +

            + You want a vehicle to protect its occupants, but you can’t just add layers and layers of titanium armor plating. You have to balance safety against weight and cost. +

            +

            + You want a phone to have the longest possible battery life, but you can’t just give it a 10-pound brick of a battery. You have to balance battery life against size and usability. +

            +

            + In the case of notes, I believe the two priorities we are trying to balance are discoverability and understanding. +

            +

            + +

            +

            + Making a note discoverable involves making it small, simple, and easy to digest. We accomplish this using compression: creating highly condensed summaries, without all the fluff. +

            +

            + But we also want to make our notes understandable. This involves including all the context: the details, the examples, and cited sources to be sure nothing falls through the cracks. +

            +

            + +

            +

            + This is a difficult tradeoff because you cannot compress something without losing some of its context. +

            +

            + You cannot summarize an article without discarding most of its points. You cannot make a highlight reel of a video without cutting out most of the footage. You cannot give an 18-minute TED talk without leaving out most of your ideas. +

            +

            + In making decisions about what to keep, you are inevitably making decisions about what to throw away. +

            +

            + Compression vs. context +

            +

            + There’s a natural tension between the two, compression and context. +

            +

            + +

            +

            + To communicate anything, you have to compress it, like communicating a huge amount of life experience in a wise saying. But in doing so, you lose a lot of the context that made that wisdom valuable in the first place. +

            +

            + Let’s look at some examples. +

            +

            + +

            +

            + If we compress a note too much, in other words, we make a summary that is too brief, we lose the context and it loses all meaning. In the note above, for example, the information it contains is highly discoverable — I can get the gist of it with just a glance. +

            +

            + But if I come across this note a year from now, I’ll have no idea what it means or why it’s important. It’s too compressed. +

            +

            + But we can go too far in the opposite direction too. If we make something totally understandable, in other words, if we include every little detail and bit of context, it loses its discoverability. +

            +

            + +

            +

            + The example above is my notes on the task management software Jira. It has lots of context, making it highly understandable. But it’s not discoverable at all. It would probably take me a couple hours and tremendous mental effort to read through this note and remember enough context to decide whether or not it’s useful. The main points and key insights are hidden somewhere in the noise. +

            +

            + Getting the balance between compression and context right is not a trivial matter. When the time comes for Future You to decide whether or not to review this note, seconds count. Because Future You will likely be looking for a solution to a problem, not casual reading, they will be making snap decisions on a tight timeline. Faced with a wall of text of questionable value, they are unlikely to take the risk of committing time for review. +

            +

            + This means that all the summarizing work your Past Self did on this note is wasted. It didn’t pay off back then, and it doesn’t pay off in the future. You successfully sent a packet of information forward through time, but not in a state where it could survive the journey. +

            +

            + Opportunistic compression +

            +

            + I’ve found that most people do just fine on the context side of the equation. We know how to take exhaustive notes on a book, a presentation, or a class. +

            +

            + Progressive Summarization focuses therefore on rebalancing the equation. It is a method for opportunistic compression — summarizing and condensing a piece of information in small spurts, spread across time, in the course of other work, and only doing as much or as little as the information deserves. +

            +

            + If you remember, compression is a means to improving discoverability. So our design challenge when creating a note is: +

            +
            +

            + “How do I make what I’m consuming right now easily discoverable for my future self?” +

            +
            +

            + This isn’t an easy question to answer, because you have no idea what Future You remembers, is interested in, or is working on. You have to summarize the note without knowing what it will be used for. It is general purpose summarization, a much greater challenge than extracting takeaways for just one specific project. +

            +

            + Progressive Summarization works in “layers” of summarization. Layer 0 is the original, full-length source text. +

            +

            + Layer 1 is the content that I initially bring into my note-taking program. I don’t have an explicit set of criteria on what to keep. I just capture anything that feels insightful, interesting, or useful. +

            +

            + This can include virtually any type of media, but for this article I will focus on text. There are many ways of doing this: +

            +
              +
            • Copy a paragraph of text from a PDF I’m reading, and paste it into the Evernote menu bar helper +
            • +
            • Type my random thoughts into a new note on the Evernote mobile app +
            • +
            • Dropping a Word document onto the Evernote icon in the dock on my Mac, which adds it to a note as an attachment +
            • +
            • Downloading all my Kindle highlights from a book using Bookcision, and then copying and pasting them into a new note +
            • +
            • Forward an email with useful information to my personal import address, which automatically imports the whole email to a note +
            • +
            • Highlight the best passages of an online article using the web highlighter Liner, which exports directly to Evernote +
            • +
            +

            + The examples above are from my recommended program Evernote (iOS, Android, Mac, Windows, browsers), but all the major note-taking platforms support the above functionality in one way or another: Bear (Mac and iOS), Simplenote (iOS, Android, Mac, Windows, Linux), Microsoft OneNote (iOS, Android, Mac, Windows), and Google Keep (browsers, iOS, Android). +

            +

            + Layer 1 is the starting point of Progressive Summarization, like the bedrock on which everything else is built: +

            +

            + +

            +

            + Layer 2 is the first round of true summarization, in which I bold only the best parts of the passages I’ve imported. Again, I have no explicit criteria. I look for keywords, key phrases, and key sentences that I feel represent the core or essence of the idea being discussed. +

            +

            + +

            +

            + I do this bolding layer at a later time, when I’m already reviewing this note anyway. I’m essentially using the attention I’m already spending for a dual purpose: to “buy” the information I need for the project at hand, and also to summarize the note for future use. If you have to pay attention to something, it comes in handy to be able to double-spend. +

            +

            + For Layer 3, I switch to highlighting, so I can make out the smaller number of highlighted passages among all the bolded ones. This time, I’m looking for the “best of the best,” only highlighting something if it is truly unique or valuable. And again, I’m only adding this third layer when I’m already reviewing the note anyway. +

            +

            + +

            +

            + For Layer 4, I’m still summarizing, but going beyond highlighting the words of others, to recording my own. For a small number of notes that are the most insightful, I summarize layers 2 and 3 in an informal executive summary at the top of the note, restating the key points in my own words. +

            +

            + +

            +

            + Note that all the previous layers are preserved in context, giving you the freedom to leave things out without worrying that you’ll lose them. Summarization is risky — you may be making the wrong decision about what’s important. But with the safety net of multiple layers of preserved notes, you can strike out decisively on daring intellectual expeditions. +

            +

            + And finally, for a tiny minority of sources, the ones that are so powerful and exciting I want them to become part of how I think and work immediately, I remix them. After pulling them apart and dissecting them from every angle in layers 1–4, I add my own personality and creativity and turn them into something else. +

            +

            + +

            +

            + This could include a blog post interpreting, critiquing, or extending the argument an author is making, such as in Strategically Constrained, The Inner Game of Work, and Supersizing the Mind. +

            +
            +
            +
            +
            +

            + Use Progressive Summarization
            + to create easy-to-review notes +

            +
            +
            +

            + I'll send you my Progressive Summarization Cheat Sheet as a thank you when you subscribe to my free weekly newsletter below. +

            +
            +
              +
              +
              + +
              +
              +
              +
              +

              + Look out for an email from hello@fortelabs.co +

              +
              +
              +
              +

              +   +

              +

              + But it doesn’t have to be difficult or time-consuming. It could even be…(gasp) fun! Making a sketch, designing a slide, recording a short video on your phone, and sharing on social media are all forms of wrestling deeply with information. +

              + +

              + +

              +
              +
              + The first tweet in a tweetstorm I wrote about the book Toyota Kata +
              +
              +

              + In Part II, we’ll look at some examples of Progressive Summarization in action. +

              +
              +

              + Follow us for the latest updates and insights around productivity and Building a Second Brain on Twitter, Facebook, Instagram, LinkedIn, and YouTube. And if you're ready to start building your Second Brain, get the book and learn the proven method to organize your digital life and unlock your creative potential.
              +
              +


              +
              +
              +
              +
              +
              +

              + Join the Forte Labs Newsletter +

              +
              +
              +

              + Every Tuesday, I send over 100,000 subscribers new essays, videos, event invites, and other resources designed to level up your productivity and life. +

              +

              + ​ +

              +

              + Join us, and I'll send you my Top 10 Most Popular Articles right away as a thank you. +

              +
              +
              +
              +
                +
                +
                + +
                +
                +
                +
                +
                +
                +
                +
                +
                + Series Navigation: Progressive Summarization +
                + + + + + + + +
                + Progressive Summarization II: Examples and Metaphors >> +
                +
                +
                +
                +
                +
                +
                +
                +
                +
                +
                +
                + +
                +
                +
                +
                +
                +
                +
                +
                + +
                +
                +
                +
                +
                +
                +
                +
                +
                + + + +
                +
                  +
                • + +
                • +
                • + +
                • +
                • + +
                • +
                • + +
                • +
                • + +
                • +
                • + +
                • +
                +
                +
                + +
                + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                + word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word +

                + mmMwWLliI0fiflO&1
                + mmMwWLliI0fiflO&1
                + mmMwWLliI0fiflO&1
                + mmMwWLliI0fiflO&1
                + mmMwWLliI0fiflO&1
                + mmMwWLliI0fiflO&1
                + mmMwWLliI0fiflO&1 + + + +
                +
                +
                +
                + +
                +
                +
                +
                + + diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/url.txt b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/url.txt new file mode 100644 index 000000000..1cbb17cab --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/url.txt @@ -0,0 +1 @@ +https://fortelabs.com/blog/progressive-summarization-a-practical-technique-for-designing-discoverable-notes/ \ No newline at end of file From 1b1cce74854e727042a339f101e17d8e4f968531 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 20 Oct 2023 15:59:17 +0800 Subject: [PATCH 075/115] disable javascript for the host --- packages/puppeteer-parse/index.js | 2 +- .../newsletters/forte-labs/expected.html | 260 ++++++++++++++++-- 2 files changed, 244 insertions(+), 18 deletions(-) diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index 78cdc87dd..64f8b91a7 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -40,7 +40,7 @@ const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) Apple const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] -const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; +const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com', 'fortelabs.com']; const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html index 67dae7ff0..99b0e8f9c 100644 --- a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html @@ -1,19 +1,245 @@
                - - - - -

                word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word


                - mmMwWLliI0fiflO&1
                - mmMwWLliI0fiflO&1
                - mmMwWLliI0fiflO&1
                - mmMwWLliI0fiflO&1
                - mmMwWLliI0fiflO&1
                - mmMwWLliI0fiflO&1
                - mmMwWLliI0fiflO&1 +
                +
                +
                +
                +
                + Series Navigation: Progressive Summarization +
                + +
                +
                +

                + +

                +

                Modern digital tools make it easy to “capture” information from a wide variety of sources. We know how to snap a picture, type out some notes, record a video, or scan a document. Getting this content from the outside world into the digital world is trivial.

                +

                It’s even easier to get content that is already digital from one app to another. We know how to copy and paste text, save an image from a webpage, archive an email attachment, or import a video file.

                +

                What is difficult is not transferring content from place to place, but transferring it through time.

                +

                You know what I mean: you read a book, investing hours of mental labor in understanding the ideas it presents. You finish the book with a feeling of triumph that you’ve gained a valuable body of knowledge.

                +

                But then what?

                +

                You may try to apply the science-based methods the book recommends, only to realize it’s not quite as clear-cut as you thought. You may try to change the way you eat, exercise, communicate, or work, trusting in the power of habits. But then the everyday demands of life come rushing back, and you forget what motivated you in the first place.

                +

                At this point, people take different paths. Some give up, labeling all “self-help” books a waste of time. Others decide it’s just a problem of remembering everything they read, and invest in fancy memorization techniques. And many people become “infovores,” force-feeding themselves endless books, articles, and courses, in the hope that something will stick.

                +

                I want to suggest an alternative to all the approaches above: what you read is good and useful and very important, you’re just reading it at the wrong time.

                +

                You’re reading about time management techniques now, but they will only be useful two years from now, when you become a manager and have much greater demands on your time.

                +

                You’re watching YouTube videos on online marketing now, but that knowledge can only be put to use in 9 months, when your new online course gets off the ground.

                +

                You’re talking to a prospect about his goals and challenges now, but when you could really use that information is next year, when he is taking bids for a huge new contract.

                +

                The challenge of knowledge is not acquiring it. In our digital world, you can acquire almost any knowledge at almost any time.

                +

                The challenge is knowing which knowledge is worth acquiring. And then building a system to forward bits of it through time, to the future situation or problem or challenge where it is most applicable, and most needed.

                +

                At that future point, when you’re applying that knowledge directly to a real-world challenge, you won’t have to worry about memorizing it, integrating it, or even fully understanding it. You will only have to apply it, and any gaps in your understanding will very quickly reveal themselves. By the time you’re done solving a real problem with it, book knowledge has become experiential knowledge. And experiential knowledge is something you carry with you forever.

                +

                This is the job of a “second brain” — an external, integrated digital repository for the things you learn and the resources from which they come. It is a storage and retrieval system, packaging bits of knowledge into discrete packets that can be forwarded to various points in time to be reviewed, utilized, or deleted.

                +
                +
                +

                +

                Use Progressive Summarization
                to create easy-to-review notes

                +

                +

                I'll send you my Progressive Summarization Cheat Sheet as a thank you when you subscribe to my free weekly newsletter below.

                +
                  +

                  Look out for an email from hello@fortelabs.co

                  +
                  +
                  +

                  In The PARA Method, I described a universal system for organizing any kind of digital information from any source. It is a “good enough” system, maintaining notes according to their actionability (which takes just a moment to determine), instead of their meaning (which is ambiguous and depends on the context).

                  +

                  The four top-level categories of PARA — Projects, Areas, Resources, and Archives — are designed to facilitate this process of forwarding knowledge through time.

                  +
                    +
                  • By placing a note in a project folder, you are essentially scheduling it for review on the short time horizon of an individual project
                  • +
                  • Notes in area folders are scheduled for less frequent review, whenever you evaluate that area of your work or life
                  • +
                  • Notes in resource folders stand ready for review if and when you decide to take action on that topic
                  • +
                  • And notes in archive folders are in “cold storage,” available if needed but not scheduled for review at any particular time
                  • +
                  +

                  Note that we have re-created the tickler file, except instead of strict time-based horizons (daily, weekly, monthly, annually), they are scheduled contingently — if X happens, when Y arrives, if I want to do Z, etc.

                  +

                  + +

                  +

                  Planning in terms of contingencies gives us all the benefits of planning and researching, without locking us into rigid routines. We have the ability to massively accelerate, using our repository of accumulated notes as rocket fuel. But the actual decision of whether or not to accelerate, and critically, in which direction, we leave to our Future Self, who is older and wiser.

                  +

                  PARA answers how these “packets of knowledge” are organized: in discrete notes, sorted into 4 categories according to actionability, and resurfaced using RandomNote.

                  +

                  But now we turn to a more fundamental question: how are these packets made? Once we capture something, how do we structure the note so that it’s easily discoverable and usable in the future? How do we make sure what we’re saving today adds value to future projects, even when we can’t predict or even imagine what those projects might be?

                  +

                  That is the job of Progressive Summarization.

                  +

                  Note-first knowledge management

                  +

                  There are two primary schools of thought on how to organize a note-taking program (or really any body of information, but I’ll use terms specific to note-taking apps):

                  +

                  + +

                  +

                  + Tagging-first approaches argue that there should be no explicit hierarchy of notes, notebooks, and stacks. Notes are envisioned as an ever-changing, virtual matrix of interconnected, free-floating ideas. Because many tags can be applied to one note, there are multiple pathways to discover any given note. Locating notes in specific notebooks and folders is seen as limiting and static. +

                  +

                  + +

                  +

                  Although tags have their uses, I don’t believe they work as a primary organizational system. In my experience, relying on tagging is too fragile and requires too much maintenance, spreading attention too uniformly across all notes whether or not they are truly valuable. The virtual matrix sounds cool and futuristic, but our minds are not made to work well with such abstract concepts — we understand placing one thing in one place intuitively and automatically.

                  +

                  The second conventional approach to organizing notes is notebook-first. This basically translates how we organize things in the physical world — in a series of discrete containers — into the digital world.

                  +

                  + +

                  +

                  Notebook-first is better than tagging-first, in my opinion, mostly because it stays out of the way. It doesn’t try to automate and encroach upon the deeply intuitive act of making connections and seeing patterns. PARA on its own is a notebook-first system.

                  +

                  But if we stopped there, it would still be woefully inadequate for an economy based on creative output. As the tagging enthusiasts correctly point out, notebooks and folders actually suppress the serendipity and randomness that is at the heart of a creative lifestyle.

                  +

                  I propose a way to break the impasse: a note-first approach.

                  +

                  + +

                  +

                  I propose we make the design of individual notes the primary factor, instead of tags or notebooks. This has many advantages:

                  +
                    +
                  • It works well with any other organizational system, without depending on them (including but not limited to tags and notebooks, if you want to use those)
                  • +
                  • It makes all work you do on your notes value-added, because you’re spending close to 100% of the time engaging directly with the content itself
                  • +
                  • It can more easily survive migrations to other devices, storage locations, and even programs, because note content is much more likely to be preserved than overarching structure
                  • +
                  • It cultivates skills (succinct communication, finding the core of an idea, visual thinking, etc.) that are inherently valuable and highly transferrable to other activities
                  • +
                  • It makes your notes more legible and useful to others (unlike your internal notebook structure, which is only for your use), promoting collaboration and sharing
                  • +
                  +

                  With a note-first approach, your notes become like individual atoms — each with its own unique properties, but ready to be assembled into elements, molecules, and compounds that are far more powerful.

                  +

                  A note-first approach to knowledge management means we have to think about design. You are, in a very real sense, designing a product for a demanding customer — Future You.

                  +

                  Future You doesn’t necessarily trust that everything Past You put into your notes is valuable. Future You is impatient and skeptical, demanding proof upfront that the time they spend reviewing notes will be worthwhile. You’ve gotta “sell them” on the idea of reviewing a given note, including all the stages any salesperson has to master: gaining attention, inspiring interest, establishing credibility, stoking desire, and making a case for action NOW.

                  +

                  As if all that wasn’t intimidating enough, you have to do this for every single note without spending any extra time. You don’t have extra time, do you?

                  +

                  Let’s start at the beginning: at the heart of every design, we are trying to balance priorities. You want one thing, but it has to be balanced against something else that you also want.

                  +

                  You want a vehicle to protect its occupants, but you can’t just add layers and layers of titanium armor plating. You have to balance safety against weight and cost.

                  +

                  You want a phone to have the longest possible battery life, but you can’t just give it a 10-pound brick of a battery. You have to balance battery life against size and usability.

                  +

                  In the case of notes, I believe the two priorities we are trying to balance are discoverability and understanding.

                  +

                  + +

                  +

                  Making a note discoverable involves making it small, simple, and easy to digest. We accomplish this using compression: creating highly condensed summaries, without all the fluff.

                  +

                  But we also want to make our notes understandable. This involves including all the context: the details, the examples, and cited sources to be sure nothing falls through the cracks.

                  +

                  + +

                  +

                  This is a difficult tradeoff because you cannot compress something without losing some of its context.

                  +

                  You cannot summarize an article without discarding most of its points. You cannot make a highlight reel of a video without cutting out most of the footage. You cannot give an 18-minute TED talk without leaving out most of your ideas.

                  +

                  In making decisions about what to keep, you are inevitably making decisions about what to throw away.

                  +

                  Compression vs. context

                  +

                  There’s a natural tension between the two, compression and context.

                  +

                  + +

                  +

                  To communicate anything, you have to compress it, like communicating a huge amount of life experience in a wise saying. But in doing so, you lose a lot of the context that made that wisdom valuable in the first place.

                  +

                  Let’s look at some examples.

                  +

                  + +

                  +

                  If we compress a note too much, in other words, we make a summary that is too brief, we lose the context and it loses all meaning. In the note above, for example, the information it contains is highly discoverable — I can get the gist of it with just a glance.

                  +

                  But if I come across this note a year from now, I’ll have no idea what it means or why it’s important. It’s too compressed.

                  +

                  But we can go too far in the opposite direction too. If we make something totally understandable, in other words, if we include every little detail and bit of context, it loses its discoverability.

                  +

                  + +

                  +

                  The example above is my notes on the task management software Jira. It has lots of context, making it highly understandable. But it’s not discoverable at all. It would probably take me a couple hours and tremendous mental effort to read through this note and remember enough context to decide whether or not it’s useful. The main points and key insights are hidden somewhere in the noise.

                  +

                  Getting the balance between compression and context right is not a trivial matter. When the time comes for Future You to decide whether or not to review this note, seconds count. Because Future You will likely be looking for a solution to a problem, not casual reading, they will be making snap decisions on a tight timeline. Faced with a wall of text of questionable value, they are unlikely to take the risk of committing time for review.

                  +

                  This means that all the summarizing work your Past Self did on this note is wasted. It didn’t pay off back then, and it doesn’t pay off in the future. You successfully sent a packet of information forward through time, but not in a state where it could survive the journey.

                  +

                  Opportunistic compression

                  +

                  I’ve found that most people do just fine on the context side of the equation. We know how to take exhaustive notes on a book, a presentation, or a class.

                  +

                  Progressive Summarization focuses therefore on rebalancing the equation. It is a method for opportunistic compression — summarizing and condensing a piece of information in small spurts, spread across time, in the course of other work, and only doing as much or as little as the information deserves.

                  +

                  If you remember, compression is a means to improving discoverability. So our design challenge when creating a note is:

                  +
                  +

                  “How do I make what I’m consuming right now easily discoverable for my future self?”

                  +
                  +

                  This isn’t an easy question to answer, because you have no idea what Future You remembers, is interested in, or is working on. You have to summarize the note without knowing what it will be used for. It is general purpose summarization, a much greater challenge than extracting takeaways for just one specific project.

                  +

                  Progressive Summarization works in “layers” of summarization. Layer 0 is the original, full-length source text.

                  +

                  Layer 1 is the content that I initially bring into my note-taking program. I don’t have an explicit set of criteria on what to keep. I just capture anything that feels insightful, interesting, or useful.

                  +

                  This can include virtually any type of media, but for this article I will focus on text. There are many ways of doing this:

                  +
                    +
                  • Copy a paragraph of text from a PDF I’m reading, and paste it into the Evernote menu bar helper +
                  • +
                  • Type my random thoughts into a new note on the Evernote mobile app +
                  • +
                  • Dropping a Word document onto the Evernote icon in the dock on my Mac, which adds it to a note as an attachment
                  • +
                  • Downloading all my Kindle highlights from a book using Bookcision, and then copying and pasting them into a new note
                  • +
                  • Forward an email with useful information to my personal import address, which automatically imports the whole email to a note
                  • +
                  • Highlight the best passages of an online article using the web highlighter Liner, which exports directly to Evernote
                  • +
                  +

                  The examples above are from my recommended program Evernote (iOS, Android, Mac, Windows, browsers), but all the major note-taking platforms support the above functionality in one way or another: Bear (Mac and iOS), Simplenote (iOS, Android, Mac, Windows, Linux), Microsoft OneNote (iOS, Android, Mac, Windows), and Google Keep (browsers, iOS, Android).

                  +

                  Layer 1 is the starting point of Progressive Summarization, like the bedrock on which everything else is built:

                  +

                  + +

                  +

                  Layer 2 is the first round of true summarization, in which I bold only the best parts of the passages I’ve imported. Again, I have no explicit criteria. I look for keywords, key phrases, and key sentences that I feel represent the core or essence of the idea being discussed.

                  +

                  + +

                  +

                  I do this bolding layer at a later time, when I’m already reviewing this note anyway. I’m essentially using the attention I’m already spending for a dual purpose: to “buy” the information I need for the project at hand, and also to summarize the note for future use. If you have to pay attention to something, it comes in handy to be able to double-spend.

                  +

                  For Layer 3, I switch to highlighting, so I can make out the smaller number of highlighted passages among all the bolded ones. This time, I’m looking for the “best of the best,” only highlighting something if it is truly unique or valuable. And again, I’m only adding this third layer when I’m already reviewing the note anyway.

                  +

                  + +

                  +

                  For Layer 4, I’m still summarizing, but going beyond highlighting the words of others, to recording my own. For a small number of notes that are the most insightful, I summarize layers 2 and 3 in an informal executive summary at the top of the note, restating the key points in my own words.

                  +

                  + +

                  +

                  Note that all the previous layers are preserved in context, giving you the freedom to leave things out without worrying that you’ll lose them. Summarization is risky — you may be making the wrong decision about what’s important. But with the safety net of multiple layers of preserved notes, you can strike out decisively on daring intellectual expeditions.

                  +

                  And finally, for a tiny minority of sources, the ones that are so powerful and exciting I want them to become part of how I think and work immediately, I remix them. After pulling them apart and dissecting them from every angle in layers 1–4, I add my own personality and creativity and turn them into something else.

                  +

                  + +

                  +

                  This could include a blog post interpreting, critiquing, or extending the argument an author is making, such as in Strategically Constrained, The Inner Game of Work, and Supersizing the Mind.

                  +
                  +
                  +

                  +

                  Use Progressive Summarization
                  to create easy-to-review notes

                  +

                  +

                  I'll send you my Progressive Summarization Cheat Sheet as a thank you when you subscribe to my free weekly newsletter below.

                  +
                    +

                    Look out for an email from hello@fortelabs.co

                    +
                    +
                    +

                    But it doesn’t have to be difficult or time-consuming. It could even be…(gasp) fun! Making a sketch, designing a slide, recording a short video on your phone, and sharing on social media are all forms of wrestling deeply with information.

                    +
                    +
                    +
                    The first tweet in a tweetstorm I wrote about the book Toyota Kata +
                    +
                    +

                    + In Part II, we’ll look at some examples of Progressive Summarization in action. +

                    +
                    +
                    +

                    Follow us for the latest updates and insights around productivity and Building a Second Brain on Twitter, Facebook, Instagram, LinkedIn, and YouTube. And if you're ready to start building your Second Brain, get the book and learn the proven method to organize your digital life and unlock your creative potential.

                    +
                    +
                    +
                    +

                    +

                    Join the Forte Labs Newsletter

                    +
                    +

                    Every Tuesday, I send over 100,000 subscribers new essays, videos, event invites, and other resources designed to level up your productivity and life. +

                    +

                    +

                    Join us, and I'll send you my Top 10 Most Popular Articles right away as a thank you.

                    +
                    +
                    +
                    +
                    +
                    +
                    + Series Navigation: Progressive Summarization +
                    + +
                    +
                    +
                    + + +
                    \ No newline at end of file From 313fd77bef760ecc94442ff6e116fb26a81dc03f Mon Sep 17 00:00:00 2001 From: sywhb Date: Fri, 20 Oct 2023 08:00:39 +0000 Subject: [PATCH 076/115] Update generated html --- packages/readabilityjs/test/index.html | 940 ++++++++++++------------- 1 file changed, 470 insertions(+), 470 deletions(-) diff --git a/packages/readabilityjs/test/index.html b/packages/readabilityjs/test/index.html index e997271df..a5c572757 100644 --- a/packages/readabilityjs/test/index.html +++ b/packages/readabilityjs/test/index.html @@ -14,22 +14,10 @@ From e60b38733d674516f29208750d48b801b43d58ab Mon Sep 17 00:00:00 2001 From: Unknown Date: Sat, 21 Oct 2023 11:07:48 +0300 Subject: [PATCH 077/115] modified package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de5c1658e..b02a1da45 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "omnivore-app", "version": "1.0.0", "description": "", - "private": true, + "private": false, "workspaces": [ "packages/*" ], From c6676cd231e5752a33127e811731021c3eb4815c Mon Sep 17 00:00:00 2001 From: 0Armaan025 Date: Sun, 22 Oct 2023 13:01:31 +0530 Subject: [PATCH 078/115] made contributing.md way better, precise and emojified --- .github/CONTRIBUTING.md | 45 ++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5324e46ef..3ef5816c6 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,40 +1,29 @@ -Guidelines -========== +📝 **Guidelines for Contribution** -When contributing to Omnivore, please follow the style of the file you -are editing. For most code running `yarn lint` should tell you if -your code meets are style. +Hello there! When contributing to Omnivore, it's essential to maintain a style consistent with the file you're editing. You can quickly check if your code aligns with our style by running `yarn lint`. Your attention to detail here is appreciated! 😊 +📜 **License** -License -======= +Omnivore is proudly licensed under the [AGPL license](https://github.com/omnivore-app/omnivore/blob/main/LICENSE). Let's keep our code open and accessible to all! -Omnivore is licensed under the [AGPL license](https://github.com/omnivore-app/omnivore/blob/main/LICENSE). +🤝 **CLA (Contributor License Agreement)** -CLA -======= +We kindly request all contributors to adhere to Omnivore's CLA, which you can review and accept at [Omnivore's CLA](https://cla-assistant.io/omnivore-app/omnivore). Your commitment to this agreement is much appreciated. -Contributions are accepted under [Omnivore's CLA](https://cla-assistant.io/omnivore-app/omnivore). +👩‍💻 **Code Review** -Code Review -======= -We require code review from a [CODEOWNER](https://github.com/omnivore-app/omnivore/blob/main/.github/CODEOWNERS) before merging a pull request. +Before merging any pull request, we require a code review from one of our trusted [CODEOWNERS](https://github.com/omnivore-app/omnivore/blob/main/.github/CODEOWNERS). It ensures the quality and integrity of our codebase. -Testing -======= +🧪 **Testing** -Pull requests are automatically tested using GitHub Actions. We -usually only merge a pull request if it causes no regressions in -our tests. +We take quality seriously! Pull requests automatically undergo testing using GitHub Actions. We typically merge a pull request only if it passes all our tests without causing any regressions. 🧪 -When you submit a pull request: +📬 **Submitting a Pull Request** -* If you are a new contributor, GitHub will ask for permissions (on - the pull request) to test it. A maintainer will reply to approve - the test run if they find the patch appropriate. -* If you have previously contributed, GitHub will test your pull - request as soon as a test machine is available. +- **New Contributors**: If you're a new contributor, GitHub may request permissions on the pull request to run tests. Don't worry, our maintainer will swiftly approve the test run if your patch looks promising. -Once tests are passing on your pull request it will be reviewed, -merged, and deployed by a maintiner. We deploy around ten times -a day, so your changes should hit production quickly. +- **Previous Contributors**: For those who've contributed before, GitHub will kick off testing as soon as a test machine is available. + +Once your pull request passes the tests, it'll be promptly reviewed, merged, and deployed by one of our diligent maintainers. We usually deploy changes around ten times a day, so your contributions will go live quickly! 🚀 + +Thank you for being part of the Omnivore community. Let's keep coding, collaborating, and creating something amazing together! 🌟 \ No newline at end of file From 1e9a19be7c1b82cdf7bc046ef472bd446259ba24 Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Mon, 23 Oct 2023 02:58:06 +0800 Subject: [PATCH 079/115] Add basic Traditional Chinese translation --- .../src/main/res/values-zh-rTW/strings.xml | 207 ++++++++++++++++++ .../zh-Hant.lproj/Localizable.strings | 206 +++++++++++++++++ 2 files changed, 413 insertions(+) create mode 100644 android/Omnivore/app/src/main/res/values-zh-rTW/strings.xml create mode 100644 apple/OmnivoreKit/Sources/Views/Resources/zh-Hant.lproj/Localizable.strings diff --git a/android/Omnivore/app/src/main/res/values-zh-rTW/strings.xml b/android/Omnivore/app/src/main/res/values-zh-rTW/strings.xml new file mode 100644 index 000000000..ff6db1f70 --- /dev/null +++ b/android/Omnivore/app/src/main/res/values-zh-rTW/strings.xml @@ -0,0 +1,207 @@ + + Omnivore + 絕不錯過精彩閱讀 + 深入了解 + 在無干擾的閱讀器中儲存文章,以便稍後閱讀。 + 標記 + 複製 + 註解 + 移除 + 標記 + 複製 + 註解 + 複製 + 註解 + + + 使用 Apple 繼續 + 正在登入... + + + 建立您的個人檔案 + 載入中... + 取消註冊 + 送出 + 名稱 + 名稱 + 使用者名稱 + 使用者名稱 + 請輸入有效的名稱和使用者名稱。 + + + 載入中... + 返回社交登入頁面 + 還沒有帳戶? + 忘記密碼? + 登入 + user@email.com + 電子郵件 + 密碼 + 密碼 + 請輸入電子郵件地址和密碼。 + + + 我們已向 %1$s 傳送驗證電子郵件。請驗證您的電子郵件,然後點選下面的按鈕。 + 檢查狀態 + 使用不同的電子郵件? + 載入中... + 返回社交登入頁面 + 已經有帳戶? + 註冊 + user@email.com + 電子郵件 + 密碼 + 密碼 + 名稱 + 名稱 + 使用者名稱 + 使用者名稱 + 請完成所有欄位。 + + + 使用 Google 繼續 + 正在登入... + + + 自建伺服器設定已更新。 + 自建伺服器設定已重設。 + 使用者名稱必須介於 4 到 15 個字元之間。 + 使用者名稱只能包含字母和數字。 + 此使用者名稱不可用。 + 抱歉,我們無法連線到伺服器。 + 出了些問題。請檢查您的電子郵件/密碼,然後再試一次。 + 出了些問題。請檢查您的登入資訊,然後再試一次。 + 無法使用 Google 進行身份驗證。 + 找不到身份驗證權杖。 + + + 載入中... + 重設 + 返回 + 儲存 + 了解更多關於自建伺服器 Omnivore 的資訊 + API 伺服器 + Web 伺服器 + 請輸入 API 伺服器和 Web 伺服器地址。 + + + 忽略 + 使用電子郵件繼續 + 自建伺服器選項 + + + 建立新標籤 + 指定名稱和顏色。 + 建立 + 取消 + 標籤名稱 + + + 按標籤篩選 + 設定標籤 + 取消 + 搜尋 + 儲存 + 建立名為 \"%1$s\" 的新標籤 + 提供的名稱太長(必須小於或等於 %1$d 個字元) + + + 標籤 + + + 圖書館 + + 搜尋 + + + 標籤已更新 + 無法設定標籤 + + + 筆記本 + 複製 + 筆記本已複製 + + + 註解 + 儲存 + 取消 + + + 文章註解 + 新增註解... + + + 標記 + 複製 + 標記已複製 + 新增註解... + 您尚未在此頁面新增任何標記。 + + + 字型大小: + 邊距 + 行距 + 主題: + 自動 + 高對比文字 + 對齊文字 + + + 我們無法取得您的內容。 + 閱讀器偏好設定 + 筆記本 + 開啟連結 + + + 在瀏覽器中開啟 + 儲存到 Omnivore + 複製連結 + 取消 + + + 連結已儲存 + 儲存連結時出錯 + 連結已複製 + + + 儲存中 + 現在閱讀 + 稍後閱讀 + 忽略 + + + 正在儲存到 Omnivore... + 您尚未登入。請在儲存前登入。 + 頁面已儲存 + 儲存您的頁面時出錯 + + + 編輯標籤 + 封存 + 取消封存 + 分享原始內容 + 移除項目 + + + 登出 + 您確定要登出嗎? + 確認 + 取消 + + + 管理帳戶 + 重設資料快取 + + + 設定 + + + 設定 + 文件 + 回饋 + 隱私政策 + 條款和條件 + 管理帳戶 + 登出 + diff --git a/apple/OmnivoreKit/Sources/Views/Resources/zh-Hant.lproj/Localizable.strings b/apple/OmnivoreKit/Sources/Views/Resources/zh-Hant.lproj/Localizable.strings new file mode 100644 index 000000000..a9dafd494 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Resources/zh-Hant.lproj/Localizable.strings @@ -0,0 +1,206 @@ +// Unit test Entry -- Do not remove this or add entries before this one. +// This allows us to check for syntax errors in this file with a unit test +"unitTestLeadingEntry" = "僅供測試用途。"; + +// share extension +"saveArticleSavedState" = "已儲存至 Omnivore"; +"saveArticleProcessingState" = "正在儲存至 Omnivore"; +"extensionAppUnauthorized" = "請先從應用程式登入 Omnivore,然後再儲存您的第一個連結。"; +"saveToOmnivore" = "儲存至 Omnivore"; + +// audio player +"audioPlayerReplay" = "重新播放"; + +// Highlights List Card +"highlightCardHighlightByOther" = "由其他人標註"; +"highlightCardNoHighlightsOnPage" = "您尚未在此頁面新增任何標註。"; + +// Labels View +"labelsViewAssignNameColor" = "指定名稱和顏色。"; +"createLabelMessage" = "建立新標籤"; +"labelsPurposeDescription" = "使用標籤來建立連結的精選集。"; +"labelNamePlaceholder" = "標籤名稱"; + +// Manage Account View +"manageAccountDelete" = "刪除帳戶"; +"manageAccountResetCache" = "重設資料快取"; +"manageAccountConfirmDeleteMessage" = "您確定要刪除您的帳戶嗎?此動作無法撤銷。"; + +// Newsletter Emails View +"newsletterEmailsExisting" = "現有電子郵件(點選以複製)"; +"createNewEmailMessage" = "建立新的電子郵件地址"; +"newslettersDescription" = "將 PDF 加入您的圖書館,或使用 Omnivore 電子郵件地址訂閱電子報。"; +"noCurrentSubscriptionsMessage" = "您目前沒有任何訂閱。"; + +// Profile View +"profileConfirmLogoutMessage" = "您確定要登出嗎?"; + +// Devices View +"devicesTokensTitle" = "已註冊的裝置權杖(滑動以移除)"; +"devicesCreated" = "已建立:"; + +// Push Notification Settings +"notificationsEnabled" = "已啟用通知"; +"notificationsExplainer" = "啟用推播通知將授予 Omnivore 裝置傳送通知的權限,\n但您可以控制哪些通知被傳送。"; +"notificationsTriggerExplainer" = "推播通知是由您的\n[帳戶規則](https://omnivore.app/settings/rules) 觸發,您可以在線上編輯。"; +"notificationsEnable" = "啟用推播通知?"; +"notificationsGeneralExplainer" = "當電子報連結抵達您的收件匣時獲得通知。或從我們的分享擴充功能接收提醒。"; +"notificationsOptionDeny" = "不,謝謝"; +"notificationsOptionEnable" = "是的,請繼續"; + +// Community Modal +"communityHeadline" = "協助建立 Omnivore 社群"; +"communityAppstoreReview" = "在 AppStore 上評價"; +"communityTweet" = "關於 Omnivore 的推文"; +"communityFollowTwitter" = "在 Twitter 上追蹤我們"; +"communityJoinDiscord" = "在 Discord 上加入我們"; +"communityStarGithub" = "在 GitHub 上給我們星星"; + +// Clubs View +"clubsLearnTitle" = "了解更多關於俱樂部的資訊"; +"clubsName" = "俱樂部名稱"; +"clubsCreate" = "建立新俱樂部"; +"clubsYours" = "您的俱樂部"; +"clubsNotAMemberMessage" = "您並非任何俱樂部的成員。\n建立新俱樂部並將邀請連結傳送給您的朋友以開始。\n\n在測試版期間,您最多可以建立三個俱樂部,每個俱樂部\n可以有最多十二個使用者。"; +"clubsErrorCopying" = "複製邀請 URL 時發生錯誤"; +"clubsAdminDenyViewing" = "此俱樂部的管理員不允許檢視所有成員。"; +"clubsNoMembers" = "此俱樂部沒有任何成員。透過傳送\n邀請連結將使用者加入您的俱樂部。"; +"clubsLeave" = "離開俱樂部"; +"clubsLeaveConfirm" = "您確定要離開此俱樂部嗎?不會刪除任何資料,但您將停止接收來自俱樂部的推薦。"; +"clubsNoneJoined" = "您沒有可以發布的俱樂部。\n加入俱樂部或建立自己的俱樂部以開始推薦文章。"; + +// Subscriptions +"subscriptionsErrorRetrieving" = "抱歉,我們無法取得您的訂閱。"; +"subscriptionsNone" = "您目前沒有任何訂閱。"; +//"subscriptions.error.retrieving" = "Last received: \(updatedDate.formatted())"; // unused for now + +// Text to Speech +"texttospeechLanguageDefault" = "預設語言"; +"texttospeechSettingsAudio" = "音訊設定"; +"texttospeechSettingsEnablePrefetch" = "啟用音訊預取"; +"texttospeechBetaSignupInProcess" = "正在報名測試版"; +"texttospeechBetaRealisticVoiceLimit" = "您正在參加超逼真語音測試版。在測試版期間,您每天可以聆聽 10,000 個單詞的音訊。"; +"texttospeechBetaRequestReceived" = "我們已收到您參加超逼真語音示範的請求。當有空位時,我們將透過電子郵件通知您。"; +"texttospeechBetaWaitlist" = "超逼真語音目前處於有限的測試版,並僅提供英語。啟用此功能將會將您加入測試版佇列。"; + +// Sign in/up +"registrationNoAccount" = "還沒有帳戶?"; +"registrationForgotPassword" = "忘記密碼?"; +"registrationStatusCheck" = "檢查狀態"; +"registrationUseDifferentEmail" = "使用不同的電子郵件?"; +"registrationFullName" = "全名"; +"registrationUsername" = "使用者名稱"; +"registrationAlreadyHaveAccount" = "已經有帳戶?"; +"registrationBio" = "個人簡介(選填)"; +"registrationWelcome" = "歡迎來到 Omnivore!"; +"registrationUsernameAssignedPrefix" = "您的使用者名稱是:"; +"registrationChangeUsername" = "更改使用者名稱"; +"registrationEdit" = "編輯"; +"googleAuthButton" = "使用 Google 繼續"; +"registrationViewSignUpHeadline" = "註冊"; +"loginErrorInvalidCreds" = "提供的登入憑證無效。"; + +// Recommendation +"recommendationToPrefix" = "至:"; +"recommendationAddNote" = "新增註解(選填)"; +//"recommendationToPrefix" = "Include your \(viewModel.highlightCount) highlight\(viewModel.highlightCount > 1 ? "s" : """; // unused for now +"recommendationError" = "推薦此頁面時發生錯誤"; + +// Web Reader +"readerCopyLink" = "複製連結"; +"readerSave" = "儲存至 Omnivore"; +"readerError" = "發生錯誤"; + +// Debug Menu +"menuDebugTitle" = "除錯選單"; +"menuDebugApiEnv" = "API 環境:"; + +// Navigation +"navigationSelectLink" = "從您的圖書館選擇一個連結"; +"navigationSelectSidebarToggle" = "切換側邊欄"; + +// Welcome View +"welcomeTitle" = "專為認真讀者設計的稍後閱讀工具。"; +"welcomeLearnMore" = "了解更多"; +"welcomeSignupAgreement" = "註冊即表示您同意 Omnivore 的\n"; +"welcomeTitleTermsOfService" = "服務條款"; +"welcomeTitleAndJoiner" = " 和 "; +"welcomeTitleEmailContinue" = "使用電子郵件繼續"; + +// Keyboard Commands +"keyboardCommandDecreaseFont" = "減小字型大小"; +"keyboardCommandIncreaseFont" = "增大字型大小"; +"keyboardCommandDecreaseMargin" = "減小邊距"; +"keyboardCommandIncreaseMargin" = "增大邊距"; +"keyboardCommandDecreaseLineSpacing" = "減小行距"; +"keyboardCommandIncreaseLineSpacing" = "增大行距"; + +// Library +//"library.by.author.suffix" = "by \(author)" // unused +//"Recommended by \(byStr) in \(inStr)" // unused + +// Generic +"genericSnooze" = "稍後提醒"; +"genericClose" = "關閉"; +"genericCreate" = "建立"; +"genericConfirm" = "確認"; +"genericProfile" = "個人資料"; +"genericNext" = "下一步"; +"genericName" = "名稱"; +"genericOk" = "確定"; +"genericRetry" = "重試"; +"genericEmail" = "電子郵件"; +"genericPassword" = "密碼"; +"genericSubmit" = "送出"; +"genericContinue" = "繼續"; +"genericSend" = "傳送"; +"genericOptions" = "選項"; +"genericOpen" = "開啟"; +"genericChangeApply" = "套用變更"; +"genericTitle" = "標題"; +"genericAuthor" = "作者"; +"genericDescription" = "描述"; +"genericSave" = "儲存"; +"genericLoading" = "載入中..."; +"genericFontFamily" = "字型家族"; +"genericHighContrastText" = "高對比文字"; +"enableHighlightOnReleaseText" = "自動標註模式"; +"enableJustifyText" = "對齊文字"; +"genericFont" = "字型"; +"genericHighlight" = "標註"; +"labelsGeneric" = "標籤"; +"emailsGeneric" = "電子郵件"; +"subscriptionsGeneric" = "訂閱"; +"textToSpeechGeneric" = "文字轉語音"; +"privacyPolicyGeneric" = "隱私權政策"; +"termsAndConditionsGeneric" = "使用條款"; +"feedbackGeneric" = "意見回饋"; +"manageAccountGeneric" = "管理帳戶"; +"logoutGeneric" = "登出"; +"doneGeneric" = "完成"; +"cancelGeneric" = "取消"; +"exportGeneric" = "匯出"; +"inboxGeneric" = "收件匣"; +"readLaterGeneric" = "稍後閱讀"; +"newslettersGeneric" = "電子報"; +"allGeneric" = "全部"; +"archivedGeneric" = "已封存"; +"highlightedGeneric" = "已標註"; +"filesGeneric" = "檔案"; +"newestGeneric" = "最新"; +"oldestGeneric" = "最舊"; +"recentlyReadGeneric" = "最近閱讀"; +"recentlyPublishedGeneric" = "最近發布"; +"clubsGeneric" = "俱樂部"; +"filterGeneric" = "篩選器"; +"errorGeneric" = "發生錯誤,請再試一次。"; +"pushNotificationsGeneric" = "推播通知"; +"dismissButton" = "關閉"; +"errorNetwork" = "我們無法連線到網路。"; +"documentationGeneric" = "文件"; + +// TODO: search navigationTitle, toggle, section, button, Label, title: ", CreateProfileViewModel, TextField, .keyboardShortcut + +// Unit test Entry -- Do not remove this or add entries after this one. +// This allows us to check for syntax errors in this file with a unit test +"unitTestTrailingEntry" = "僅供測試用途。"; From cfe065d47b88f20d1b78cf9fa9a720d0767e61d0 Mon Sep 17 00:00:00 2001 From: ismail Date: Sun, 22 Oct 2023 19:43:05 -0400 Subject: [PATCH 080/115] Add arabic guide --- .../ar/getting-started-guide.md | 178 ++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 docs/guides/getting-started/ar/getting-started-guide.md diff --git a/docs/guides/getting-started/ar/getting-started-guide.md b/docs/guides/getting-started/ar/getting-started-guide.md new file mode 100644 index 000000000..9338fa29f --- /dev/null +++ b/docs/guides/getting-started/ar/getting-started-guide.md @@ -0,0 +1,178 @@ + +# الشروع في إستعمال أمنيفور + +تطبيقة أومنيفور هي تطبيقة للقراءة لاحقاً هي تطبيقة للقراءة لاحقاً تتيح لك من تسجيل وتنظيم كل ما تقرأه على الإنترنت + +هذا الدليل سيوضح لك كيفية استخدام الوظائف الأساسية والميزات المتقدمة في أومنيفور، مقسمة إلى أربعة نشاطات رئيسية + +- حفض +- قراءة +- تنظيم +- اندماجات + + +المكتبة هي مركز تجربتك في أومنيفور، حيث يمكنك الوصول بسرعة إلى أي روابط قمت بحفظها. +الروابط المحفوظة تبقى في مكتبتك إلى الأبد ما لم تقم بحذفها. + +## حفظ + +هناك خمس طرق رئيسية لحفظ الروابط للصفحات أو المقالات التي ترغب في قراءتها لاحقًا: + +1 - الحفظ من مكتبة أومنيفور الخاصة بك +2 - الحفظ من متصفح +3 - الحفظ من هاتف أو جهاز لوحي (آي أو إس أو أندرويد) +4 - الاشتراك في النشرات الإخبارية عبر البريد الإلكتروني +5 - حفظ ملفات بي دي إف من جهاز ماك + + +### الحفض من مكتبة أمنيفور + +1 - في الزاوية العليا اليمنى من مكتبتك، انقر على زر إضافة رابط. +2- أدخل العنوان الذي ترغب في حفظه وانقر على إضافة رابط. +3 - سيظهر الرابط في مكتبتك في المرة التالية التي تقوم فيها بتحديثها. + + +### الحفظ من المتصفح + +1 - قم بتحميل وتثبيت إضافة أومنيفور لمتصفحك: + +- [كروم ](https://omnivore.app/install/chrome) +- [ايدج](https://omnivore.app/install/edge) +- [فاير فوكس](https://omnivore.app/install/firefox) +- [سفري](https://omnivore.app/install/safari) + +2 -انتقل إلى الصفحة التي ترغب في حفظها وانقر على زر أومنيفور في شريط أدوات المتصفح أو قائمة الإضافات. +3 - أو بدلاً عن ذلك يمكنك النقر بزر فأرة الحاسوب الأيمن على أي رابط تشعبي أو اختر **حفظ إلى أمنيفور** من القائمة +4 - سيظهر الرابط في مكتبتك في المرة التالية التي تقوم فيها بتحديثها + + +### الحفظ من الهاتف أو الجهاز اللوحي + +أفضل طريقة لحفظ الروابط من جهازك المحمول هي عبر تطبيق أمنيفور. يمكنك تحميل التطبيق هنا: + + +أي أو إس (ايباد أو أيفون) (https://omnivore.app/install/ios) +أندرويد + + +بعد تثبيت تطبيق الهاتف المحمول: + +1 - في المتصفح الخاص بك، انتقل إلى الصفحة التي ترغب في حفظها واضغط على زر شارك +2 - اضغط على أيقونة أومنيفور في قائمة الشارك +3 - سيظهر الرابط في مكتبتك عندما تقوم بتحديثها في المرة القادمة + + +### الاشتراكات في النشرة الإخبارية عبر البريد الإلكتروني + +1 - على موقع أو تطبيق أومنيفور، اضغط على صورتك أو الحروف الأولى أو الصورة الرمزية في الزاوية العليا اليمنى للوصول إلى قائمة الملف الشخصي. + اختر البريد الإلكتروني من القائمة. + 2 - اضغط على إنشاء عنوان بريد إلكتروني جديد لإضافة عنوان بريد إلكتروني جديد (مثل: username-123_abc@inbox.omnivore.app) إلى القائمة. + 3 - انقر على أيقونة النسخ بجوار عنوان البريد الإلكتروني + 4 - انتقل إلى صفحة التسجيل للنشرةgetting-started-guide.md الإخبارية التي ترغب في الاشتراك فيها + 5 - الصق عنوان البريد الإلكتروني أومنيفور في نموذج التسجيل + 6 - سيتم توصيل النشرات الإخبارية الجديدة تلقائيًا إلى صندوق الوارد أومنيفور الخاص بك + + +### حفظ ملفات البي دي إيف من جهاز الماك + +1 - قم بتثبيت تطبيقة الماك(https://omnivore.app/install/mac) +2 - قم بتحديد موقع ملف البي دي إف الذي ترغب في حفظه ثم انقر على إسم الملف بالزر الايمن من فأرة الحاسوب +3 - اختر شارك من القائمة واختر أومنيفور +4 - سيظهر الرابط في مكتبتك عندما تقوم بتحديثها في المرة القادمة + + +القراءة ## + +انقر على أي رابط محفوظ في مكتبتك لدخول وضع القراءة +تقوم أومنيفور بتنسيق الصفحات لتسهيل القراءة والتظليل، وإزالة الإعلانات والتلهية لقراءة خالية من التشتيت. كما يجعل وضع التركيز على نص المقالات أصغر حجمًا وأسرع في التحميل. +أثناء القراءة، يمكنك: +- تغيير التنسيق +- تسليط الضوء على النص +- إضافة ملاحظات +- عرض كافة النقاط البارزة والملاحظات المحفوظة +- تتبع تقدم القراءة + +### تغيير التنسيق + +1 - **_المظهر_**: اضغط على صورتك، أو الحروف الأولى، أو الصورة الرمزية في ا +لزاوية العليا اليمنى للوصول إلى قائمة الملف الشخصي. اختر الصورة المصغرة البيضاء أو السوداء لاختيار مظهر فاتح أو مظهر داكن + +2 - **_ تنسيق النص_** : اضغط على ايقونة Aa لتعديل حجم النص، الخط،الهوامش وتباعد الأسطر + + +### تسليط الضوء على النص + +1 - حدد النص الذي ترغب في تسليط الضوء عليه +2 - اضغط على زر تسليط الضوء +3 - سيظهر النص مظللاً عندما تعاود قراءة المقال في المرة القادمة + + +### إضافة ملاحظات + +1 - قم بتظليل قسم من النص حيث ترغب في إضافة ملاحظة +2 - اضغط على زر ملاحظة، اكتب ملاحظتك، ثم اضغط على حفظ +3 - ستظهر أيقونة الملاحظة عندما تعيد قراءة هذا المقال في المرة القادمة + +### عرض جميع النصوص المظللة والملاحظات المحفوظة +1 - اضغط على أيقونة تظليل/ملاحظة لرؤية قائمة بجميع النصوص المظللة والملاحظات التي أضفتها إلى هذه الصفحة. +2 - لإزالة ملاحظة أو تظليل، اخترها من القائمة ثم اضغط على أيقونة السلة + + + ### تتبع تقدم القراءة + + يقوم أومنيفور بتتبع تقدم القراءة الخاص بك تلقائيًا + عبر أجهزتك المختلفة بحيث يمكنك استئناف القراءة من حيث توقفت بسهولة. سيظهر شريط التقدم في أعلى كل رابط في مكتبتك بعد بدء القراءة. + + +## تنظيم + +بشكل تلقائي، يعرض صندوق الوارد في المكتبة جميع الروابط التي قد قمت بحفظها. لإدارة قائمتك والحفاظ على تنظيم قراءتك، يوفر أمنيفور الإجراءات التالية: + +- الأرشفة +- التسميات +- البحث +- الفلاتر + +### الأرشفة + +1 - اضغط على أيقونة القائمة بجوار الرابط الذي ترغب في أرشفته (على تطبيق الجوال، اضغط مع الاستمرار على الرابط لفتح القائمة). +2 - اختر **أرشفة** +3 - الرابط سيختفي من المكتبة التلقائية لكن سيضهر إذا اخترت فلتر الارشفة(إنظر إلى الفلاتر ) + +### التسميات + +1 - اضغط على أيقونة القائمة بجوار أي رابط واختر تعيين التسميات. +2 - اختر تسمية موجودة من القائمة أو اضغط على تعديل التسميات لإنشاء تسمية جديدة +3 - ستظهر التسمية بجوار الرابط في مكتبتك. اضغط عليها لعرض جميع الروابط التي تحمل نفس التسمية +4 - على الهاتف المحمول فقط: اضغط على التسميات لرؤية قائمة جميع التسميات التي استخدمتها . اضغط على واحدة لعرض جميع الروابط التي تحمل نفس التسمية +5 - ملاحظة : أمنيفور سيقوم بتعيين بعض التسميات اتوماتيكيا مثل "Newsletters" + +### البحث + +1 - للبحث في جميع الروابط التي قمت بحفظها، أدخل كلمة أو عبارة في شريط البحث +2 - يمكنك دمج الكلمات الرئيسية مع التسميات والفلاتر لتركيز بحثك بشكل أكبر[تعرف اكثر على البحث المتقدم ](https://docs.omnivore.app/using/searcl) + + +الفيلتر ### + +1 - استخدم قائمة **فلتر** لتحسين عرض المكتبة الخاصة بك (قد تكون بعض فلتر مرئية بشكل طبيعي) +2 - اختر **القراءة لاحقاً** لعرض قائمة بجميع الروابط الغير مؤرشفة باستثناء النشرات الإخبارية +3 - اختر **التمييزات** لعرض النصوص التي قمت بتمييزها في جميع الصفحات المحفوظة +4 - اختر **هذا اليوم ** لعرض قائمة الروابط التي قمت بحفظها اليوم +5 - اختر **النشرات الإخبارية** لعرض الروابط المحفوظة عبر اشتراكاتك في النشرات الإخبارية + + +الاندماجات ### +أمنيفور يتيح لك إستعمال اندماجات بقواعد المعرفة وتطبيقات تدوين الملاحظات مثل: +- لغس سيك +- ويب هوكس + +لغس سيك ### +بفضل وصلة لغس سيك، يمكنك مزامنة جميع المقالات والتمييزات والملاحظات إلى لغس سيك ، قاعدة معرفة شهيرة. +للحصول على معلومات حول كيفية إعداد واستخدام لغس سيك يرجى الرجوع لهذا الدليل [Omnivore for Logseq Plugin Guide](https://briansunter.com/graph/#/page/omnivore-logseq-guide) + + +### ويب هوكس +يمكن لي أمنيفور تشغيل الويب هوكس عندما تحفض رابط تقروه + مثال : This example يريك إستعمال ويب هوكس لكتابة كل الروابط المحفوظة على جوجل شيتس المسجلة على جوجل درايف + \ No newline at end of file From 5862e593978f64d94ec41b07b4d47c401acb1aac Mon Sep 17 00:00:00 2001 From: ismailghedamsi Date: Sun, 22 Oct 2023 19:56:42 -0400 Subject: [PATCH 081/115] Update getting-started-guide.md Fix formatting --- .../ar/getting-started-guide.md | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/docs/guides/getting-started/ar/getting-started-guide.md b/docs/guides/getting-started/ar/getting-started-guide.md index 9338fa29f..ebf411e68 100644 --- a/docs/guides/getting-started/ar/getting-started-guide.md +++ b/docs/guides/getting-started/ar/getting-started-guide.md @@ -1,7 +1,7 @@ # الشروع في إستعمال أمنيفور -تطبيقة أومنيفور هي تطبيقة للقراءة لاحقاً هي تطبيقة للقراءة لاحقاً تتيح لك من تسجيل وتنظيم كل ما تقرأه على الإنترنت +تطبيقة أومنيفور هي تطبيقة للقراءة لاحقاً تتيح لك من تسجيل وتنظيم كل ما تقرأه على الإنترنت هذا الدليل سيوضح لك كيفية استخدام الوظائف الأساسية والميزات المتقدمة في أومنيفور، مقسمة إلى أربعة نشاطات رئيسية @@ -18,18 +18,18 @@ هناك خمس طرق رئيسية لحفظ الروابط للصفحات أو المقالات التي ترغب في قراءتها لاحقًا: -1 - الحفظ من مكتبة أومنيفور الخاصة بك -2 - الحفظ من متصفح -3 - الحفظ من هاتف أو جهاز لوحي (آي أو إس أو أندرويد) -4 - الاشتراك في النشرات الإخبارية عبر البريد الإلكتروني -5 - حفظ ملفات بي دي إف من جهاز ماك +1 - الحفظ من مكتبة أومنيفور الخاصة بك +2 - الحفظ من متصفح +3 - الحفظ من هاتف أو جهاز لوحي (آي أو إس أو أندرويد) +4 - الاشتراك في النشرات الإخبارية عبر البريد الإلكتروني +5 - حفظ ملفات بي دي إف من جهاز ماك ### الحفض من مكتبة أمنيفور -1 - في الزاوية العليا اليمنى من مكتبتك، انقر على زر إضافة رابط. -2- أدخل العنوان الذي ترغب في حفظه وانقر على إضافة رابط. -3 - سيظهر الرابط في مكتبتك في المرة التالية التي تقوم فيها بتحديثها. +1 - في الزاوية العليا اليمنى من مكتبتك، انقر على زر إضافة رابط. +2- أدخل العنوان الذي ترغب في حفظه وانقر على إضافة رابط. +3 - سيظهر الرابط في مكتبتك في المرة التالية التي تقوم فيها بتحديثها. ### الحفظ من المتصفح @@ -41,9 +41,9 @@ - [فاير فوكس](https://omnivore.app/install/firefox) - [سفري](https://omnivore.app/install/safari) -2 -انتقل إلى الصفحة التي ترغب في حفظها وانقر على زر أومنيفور في شريط أدوات المتصفح أو قائمة الإضافات. -3 - أو بدلاً عن ذلك يمكنك النقر بزر فأرة الحاسوب الأيمن على أي رابط تشعبي أو اختر **حفظ إلى أمنيفور** من القائمة -4 - سيظهر الرابط في مكتبتك في المرة التالية التي تقوم فيها بتحديثها +2 -انتقل إلى الصفحة التي ترغب في حفظها وانقر على زر أومنيفور في شريط أدوات المتصفح أو قائمة الإضافات. +3 - أو بدلاً عن ذلك يمكنك النقر بزر فأرة الحاسوب الأيمن على أي رابط تشعبي أو اختر **حفظ إلى أمنيفور** من القائمة. +4 - سيظهر الرابط في مكتبتك في المرة التالية التي تقوم فيها بتحديثها. ### الحفظ من الهاتف أو الجهاز اللوحي @@ -57,31 +57,31 @@ بعد تثبيت تطبيق الهاتف المحمول: -1 - في المتصفح الخاص بك، انتقل إلى الصفحة التي ترغب في حفظها واضغط على زر شارك -2 - اضغط على أيقونة أومنيفور في قائمة الشارك -3 - سيظهر الرابط في مكتبتك عندما تقوم بتحديثها في المرة القادمة +1 - في المتصفح الخاص بك، انتقل إلى الصفحة التي ترغب في حفظها واضغط على زر شارك +2 - اضغط على أيقونة أومنيفور في قائمة الشارك +3 - سيظهر الرابط في مكتبتك عندما تقوم بتحديثها في المرة القادمة ### الاشتراكات في النشرة الإخبارية عبر البريد الإلكتروني -1 - على موقع أو تطبيق أومنيفور، اضغط على صورتك أو الحروف الأولى أو الصورة الرمزية في الزاوية العليا اليمنى للوصول إلى قائمة الملف الشخصي. - اختر البريد الإلكتروني من القائمة. - 2 - اضغط على إنشاء عنوان بريد إلكتروني جديد لإضافة عنوان بريد إلكتروني جديد (مثل: username-123_abc@inbox.omnivore.app) إلى القائمة. - 3 - انقر على أيقونة النسخ بجوار عنوان البريد الإلكتروني - 4 - انتقل إلى صفحة التسجيل للنشرةgetting-started-guide.md الإخبارية التي ترغب في الاشتراك فيها - 5 - الصق عنوان البريد الإلكتروني أومنيفور في نموذج التسجيل - 6 - سيتم توصيل النشرات الإخبارية الجديدة تلقائيًا إلى صندوق الوارد أومنيفور الخاص بك +1 - على موقع أو تطبيق أومنيفور، اضغط على صورتك أو الحروف الأولى أو الصورة الرمزية في الزاوية العليا اليمنى للوصول إلى قائمة الملف الشخصي. + اختر البريد الإلكتروني من القائمة. + 2 - اضغط على إنشاء عنوان بريد إلكتروني جديد لإضافة عنوان بريد إلكتروني جديد (مثل: username-123_abc@inbox.omnivore.app) إلى القائمة. + 3 - انقر على أيقونة النسخ بجوار عنوان البريد الإلكتروني + 4 - انتقل إلى صفحة التسجيل للنشرةgetting-started-guide.md الإخبارية التي ترغب في الاشتراك فيها + 5 - الصق عنوان البريد الإلكتروني أومنيفور في نموذج التسجيل + 6 - سيتم توصيل النشرات الإخبارية الجديدة تلقائيًا إلى صندوق الوارد أومنيفور الخاص بك ### حفظ ملفات البي دي إيف من جهاز الماك -1 - قم بتثبيت تطبيقة الماك(https://omnivore.app/install/mac) -2 - قم بتحديد موقع ملف البي دي إف الذي ترغب في حفظه ثم انقر على إسم الملف بالزر الايمن من فأرة الحاسوب -3 - اختر شارك من القائمة واختر أومنيفور -4 - سيظهر الرابط في مكتبتك عندما تقوم بتحديثها في المرة القادمة +1 - قم بتثبيت تطبيقة الماك(https://omnivore.app/install/mac) +2 - قم بتحديد موقع ملف البي دي إف الذي ترغب في حفظه ثم انقر على إسم الملف بالزر الايمن من فأرة الحاسوب +3 - اختر شارك من القائمة واختر أومنيفور +4 - سيظهر الرابط في مكتبتك عندما تقوم بتحديثها في المرة القادمة -القراءة ## +## القراءة انقر على أي رابط محفوظ في مكتبتك لدخول وضع القراءة تقوم أومنيفور بتنسيق الصفحات لتسهيل القراءة والتظليل، وإزالة الإعلانات والتلهية لقراءة خالية من التشتيت. كما يجعل وضع التركيز على نص المقالات أصغر حجمًا وأسرع في التحميل. @@ -102,16 +102,16 @@ ### تسليط الضوء على النص -1 - حدد النص الذي ترغب في تسليط الضوء عليه -2 - اضغط على زر تسليط الضوء -3 - سيظهر النص مظللاً عندما تعاود قراءة المقال في المرة القادمة +1 - حدد النص الذي ترغب في تسليط الضوء عليه +2 - اضغط على زر تسليط الضوء +3 - سيظهر النص مظللاً عندما تعاود قراءة المقال في المرة القادمة ### إضافة ملاحظات -1 - قم بتظليل قسم من النص حيث ترغب في إضافة ملاحظة -2 - اضغط على زر ملاحظة، اكتب ملاحظتك، ثم اضغط على حفظ -3 - ستظهر أيقونة الملاحظة عندما تعيد قراءة هذا المقال في المرة القادمة +1 - قم بتظليل قسم من النص حيث ترغب في إضافة ملاحظة +2 - اضغط على زر ملاحظة، اكتب ملاحظتك، ثم اضغط على حفظ +3 - ستظهر أيقونة الملاحظة عندما تعيد قراءة هذا المقال في المرة القادمة ### عرض جميع النصوص المظللة والملاحظات المحفوظة 1 - اضغط على أيقونة تظليل/ملاحظة لرؤية قائمة بجميع النصوص المظللة والملاحظات التي أضفتها إلى هذه الصفحة. @@ -135,39 +135,39 @@ ### الأرشفة -1 - اضغط على أيقونة القائمة بجوار الرابط الذي ترغب في أرشفته (على تطبيق الجوال، اضغط مع الاستمرار على الرابط لفتح القائمة). -2 - اختر **أرشفة** +1 - اضغط على أيقونة القائمة بجوار الرابط الذي ترغب في أرشفته (على تطبيق الجوال، اضغط مع الاستمرار على الرابط لفتح القائمة) +2 - اختر **أرشفة** 3 - الرابط سيختفي من المكتبة التلقائية لكن سيضهر إذا اخترت فلتر الارشفة(إنظر إلى الفلاتر ) ### التسميات -1 - اضغط على أيقونة القائمة بجوار أي رابط واختر تعيين التسميات. -2 - اختر تسمية موجودة من القائمة أو اضغط على تعديل التسميات لإنشاء تسمية جديدة -3 - ستظهر التسمية بجوار الرابط في مكتبتك. اضغط عليها لعرض جميع الروابط التي تحمل نفس التسمية -4 - على الهاتف المحمول فقط: اضغط على التسميات لرؤية قائمة جميع التسميات التي استخدمتها . اضغط على واحدة لعرض جميع الروابط التي تحمل نفس التسمية -5 - ملاحظة : أمنيفور سيقوم بتعيين بعض التسميات اتوماتيكيا مثل "Newsletters" +1 - اضغط على أيقونة القائمة بجوار أي رابط واختر تعيين التسميات. +2 - اختر تسمية موجودة من القائمة أو اضغط على تعديل التسميات لإنشاء تسمية جديدة +3 - ستظهر التسمية بجوار الرابط في مكتبتك. اضغط عليها لعرض جميع الروابط التي تحمل نفس التسمية +4 - على الهاتف المحمول فقط: اضغط على التسميات لرؤية قائمة جميع التسميات التي استخدمتها . اضغط على واحدة لعرض جميع الروابط التي تحمل نفس التسمية +5 - ملاحظة : أمنيفور سيقوم بتعيين بعض التسميات اتوماتيكيا مثل "Newsletters" ### البحث -1 - للبحث في جميع الروابط التي قمت بحفظها، أدخل كلمة أو عبارة في شريط البحث -2 - يمكنك دمج الكلمات الرئيسية مع التسميات والفلاتر لتركيز بحثك بشكل أكبر[تعرف اكثر على البحث المتقدم ](https://docs.omnivore.app/using/searcl) +1 - للبحث في جميع الروابط التي قمت بحفظها، أدخل كلمة أو عبارة في شريط البحث +2 - يمكنك دمج الكلمات الرئيسية مع التسميات والفلاتر لتركيز بحثك بشكل أكبر[تعرف اكثر على البحث المتقدم ](https://docs.omnivore.app/using/searcl) -الفيلتر ### +### الفيلتر -1 - استخدم قائمة **فلتر** لتحسين عرض المكتبة الخاصة بك (قد تكون بعض فلتر مرئية بشكل طبيعي) -2 - اختر **القراءة لاحقاً** لعرض قائمة بجميع الروابط الغير مؤرشفة باستثناء النشرات الإخبارية -3 - اختر **التمييزات** لعرض النصوص التي قمت بتمييزها في جميع الصفحات المحفوظة -4 - اختر **هذا اليوم ** لعرض قائمة الروابط التي قمت بحفظها اليوم -5 - اختر **النشرات الإخبارية** لعرض الروابط المحفوظة عبر اشتراكاتك في النشرات الإخبارية +1 - استخدم قائمة **فلتر** لتحسين عرض المكتبة الخاصة بك (قد تكون بعض فلتر مرئية بشكل طبيعي) +2 - اختر **القراءة لاحقاً** لعرض قائمة بجميع الروابط الغير مؤرشفة باستثناء النشرات الإخبارية +3 - اختر **التمييزات** لعرض النصوص التي قمت بتمييزها في جميع الصفحات المحفوظة +4 - اختر **هذا اليوم ** لعرض قائمة الروابط التي قمت بحفظها اليوم +5 - اختر **النشرات الإخبارية** لعرض الروابط المحفوظة عبر اشتراكاتك في النشرات الإخبارية -الاندماجات ### +### الاندماجات أمنيفور يتيح لك إستعمال اندماجات بقواعد المعرفة وتطبيقات تدوين الملاحظات مثل: - لغس سيك - ويب هوكس -لغس سيك ### +### لغس سيك بفضل وصلة لغس سيك، يمكنك مزامنة جميع المقالات والتمييزات والملاحظات إلى لغس سيك ، قاعدة معرفة شهيرة. للحصول على معلومات حول كيفية إعداد واستخدام لغس سيك يرجى الرجوع لهذا الدليل [Omnivore for Logseq Plugin Guide](https://briansunter.com/graph/#/page/omnivore-logseq-guide) @@ -175,4 +175,4 @@ ### ويب هوكس يمكن لي أمنيفور تشغيل الويب هوكس عندما تحفض رابط تقروه مثال : This example يريك إستعمال ويب هوكس لكتابة كل الروابط المحفوظة على جوجل شيتس المسجلة على جوجل درايف - \ No newline at end of file + From 7763a8a939bcc2aa478c77d93bc81cbc5aea18d2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 10:52:03 +0800 Subject: [PATCH 082/115] perf: limit search item results to 100 a time and fetch labels and recommendations only needed --- packages/api/src/resolvers/article/index.ts | 22 ++------ .../api/src/resolvers/function_resolvers.ts | 52 ++++++++++++++++--- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index bd016a345..54b9fcfba 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -57,6 +57,7 @@ import { findHighlightsByLibraryItemId } from '../../services/highlights' import { addLabelsToLibraryItem, findLabelsByIds, + findLabelsByLibraryItemId, findOrCreateLabels, saveLabelsInLibraryItem, } from '../../services/labels' @@ -69,6 +70,7 @@ import { updateLibraryItemReadingProgress, updateLibraryItems, } from '../../services/library_item' +import { findRecommendationsByLibraryItemId } from '../../services/recommendation' import { parsedContentToLibraryItem } from '../../services/save_page' import { findUploadFileById, @@ -610,7 +612,7 @@ export const searchResolver = authorized< QuerySearchArgs >(async (_obj, params, { log, uid }) => { const startCursor = params.after || '' - const first = params.first || 10 + const first = Math.min(params.first || 10, 100) // limit to 100 items // the query size is limited to 255 characters if (params.query && params.query.length > 255) { @@ -641,25 +643,7 @@ export const searchResolver = authorized< libraryItems.pop() } - await Promise.all( - libraryItems.map(async (libraryItem) => { - if ( - libraryItem.highlightAnnotations && - libraryItem.highlightAnnotations.length > 0 - ) { - // fetch highlights for each item - libraryItem.highlights = await findHighlightsByLibraryItemId( - libraryItem.id, - uid - ) - } - }) - ) - const edges = libraryItems.map((libraryItem) => { - if (libraryItem.siteIcon && !isBase64Image(libraryItem.siteIcon)) { - libraryItem.siteIcon = createImageProxyUrl(libraryItem.siteIcon, 128, 128) - } if (params.includeContent && libraryItem.readableContent) { // convert html to the requested format const format = params.format || ArticleFormat.Html diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index c2cc34f9a..bc58b54bf 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -6,15 +6,19 @@ import { Subscription } from '../entity/subscription' import { Article, + Highlight, Label, PageType, Recommendation, SearchItem, } from '../generated/graphql' +import { findHighlightsByLibraryItemId } from '../services/highlights' import { findLabelsByLibraryItemId } from '../services/labels' import { findRecommendationsByLibraryItemId } from '../services/recommendation' import { findUploadFileById } from '../services/upload_file' import { + highlightDataToHighlight, + isBase64Image, recommandationDataToRecommendation, validatedDate, wordsCount, @@ -483,30 +487,64 @@ export const functionResolvers = { if (item.wordCount) return item.wordCount return item.content ? wordsCount(item.content) : undefined }, + siteIcon(item: { siteIcon?: string }) { + if (item.siteIcon && !isBase64Image(item.siteIcon)) { + return createImageProxyUrl(item.siteIcon, 128, 128) + } + + return item.siteIcon + }, + async highlights( + item: { + id: string + highlights?: Highlight[] + highlightAnnotations?: string[] | null + }, + _: unknown, + ctx: WithDataSourcesContext + ) { + if (item.highlights) return item.highlights + + if (item.highlightAnnotations && item.highlightAnnotations.length > 0) { + const highlights = await findHighlightsByLibraryItemId(item.id, ctx.uid) + return highlights.map(highlightDataToHighlight) + } + + return [] + }, async labels( - item: { id: string; labels?: Label[] }, + item: { id: string; labels?: Label[]; labelNames?: string[] | null }, _: unknown, ctx: WithDataSourcesContext ) { if (item.labels) return item.labels - return findLabelsByLibraryItemId(item.id, ctx.uid) + if (item.labelNames && item.labelNames.length > 0) { + return findLabelsByLibraryItemId(item.id, ctx.uid) + } + + return [] }, async recommendations( item: { id: string recommendations?: Recommendation[] + recommenderNames?: string[] | null }, _: unknown, ctx: WithDataSourcesContext ) { if (item.recommendations) return item.recommendations - const recommendations = await findRecommendationsByLibraryItemId( - item.id, - ctx.uid - ) - return recommendations.map(recommandationDataToRecommendation) + if (item.recommenderNames && item.recommenderNames.length > 0) { + const recommendations = await findRecommendationsByLibraryItemId( + item.id, + ctx.uid + ) + return recommendations.map(recommandationDataToRecommendation) + } + + return [] }, }, Subscription: { From 850569d7e4d67e9cef41cfc2b1c748cc21cec3e2 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 12:24:19 +0800 Subject: [PATCH 083/115] add test cases --- packages/api/src/services/groups.ts | 4 ++ packages/api/test/resolvers/article.test.ts | 67 +++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/packages/api/src/services/groups.ts b/packages/api/src/services/groups.ts index 0a3de5891..e3ec84ebe 100644 --- a/packages/api/src/services/groups.ts +++ b/packages/api/src/services/groups.ts @@ -280,3 +280,7 @@ export const getGroupsWhereUserCanPost = async ( .innerJoinAndSelect('members.user', 'user') .getMany() } + +export const deleteGroup = async (groupId: string) => { + return getRepository(Group).delete(groupId) +} diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 083bd389e..7cb3cdcbe 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -4,6 +4,7 @@ import chaiString from 'chai-string' import 'mocha' import sinon from 'sinon' import { DeepPartial } from 'typeorm' +import { Group } from '../../src/entity/groups/group' import { Highlight } from '../../src/entity/highlight' import { Label } from '../../src/entity/label' import { LibraryItem, LibraryItemState } from '../../src/entity/library_item' @@ -18,6 +19,7 @@ import { UploadFileStatus, } from '../../src/generated/graphql' import { getRepository } from '../../src/repository' +import { createGroup, deleteGroup } from '../../src/services/groups' import { createHighlight } from '../../src/services/highlights' import { createLabel, @@ -153,6 +155,13 @@ const searchQuery = (keyword = '') => { highlights { id } + labels { + id + name + } + recommendations { + name + } } } pageInfo { @@ -1455,6 +1464,64 @@ describe('Article API', () => { }) } ) + + context('when recommendedBy:* is in the query', () => { + let items: LibraryItem[] = [] + let group: Group + + before(async () => { + keyword = 'recommendedBy:*' + + group = ( + await createGroup({ + admin: user, + name: 'test group', + }) + )[0] + + // Create some test items + items = await createLibraryItems( + [ + { + user, + title: 'test title 1', + readableContent: '

                    test 1

                    ', + slug: 'test slug 1', + originalUrl: `${url}/test1`, + recommendations: [ + { + recommender: user, + group, + }, + ], + }, + { + user, + title: 'test title 2', + readableContent: '

                    test 2

                    ', + slug: 'test slug 2', + originalUrl: `${url}/test2`, + }, + ], + user.id + ) + }) + + after(async () => { + await deleteLibraryItems(items, user.id) + await deleteGroup(group.id) + }) + + it('returns recommended items', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.pageInfo.totalCount).to.eq(1) + expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id) + expect( + res.body.data.search.edges[0].node.recommendations[0].name + ).to.eq(group.name) + }) + }) }) describe('TypeaheadSearch API', () => { From 3b8705e4c7dab0c360f2f551d1af70d2ddbbdc9d Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 23 Oct 2023 14:24:42 +0800 Subject: [PATCH 084/115] Add library items to /admin --- pkg/admin/src/db.ts | 44 ++++++++++++++++++++++++++++++++++++++++++ pkg/admin/src/index.ts | 2 ++ 2 files changed, 46 insertions(+) diff --git a/pkg/admin/src/db.ts b/pkg/admin/src/db.ts index 471ddf9bb..df4f0830c 100644 --- a/pkg/admin/src/db.ts +++ b/pkg/admin/src/db.ts @@ -50,6 +50,7 @@ export const registerDatabase = async (secrets: any): Promise => { Group, Integration, Subscription, + LibraryItem, ], }) @@ -308,3 +309,46 @@ export class Subscription extends BaseEntity { @Column({ type: 'timestamp', name: 'updated_at' }) updatedAt!: Date } + +@Entity({ name: 'library_item' }) +export class LibraryItem { + @PrimaryGeneratedColumn('uuid') + id!: string + + @JoinColumn({ name: 'user_id' }) + @ManyToOne(() => User, (user) => user.articles, { eager: true }) + user!: User + + @Column('text') + originalUrl!: string + + @Column('text') + slug!: string + + @Column('text') + title!: string + + @Column('text', { nullable: true }) + author?: string | null + + @Column('text', { nullable: true }) + siteName?: string | null + + @Column('text') + itemType!: string + + @Column('text', { nullable: true }) + subscription?: string | null + + @Column({ type: 'timestamp', name: 'saved_at' }) + savedAt!: Date + + @Column({ type: 'timestamp', name: 'deleted_at' }) + deletedAt?: Date | null + + @Column({ type: 'timestamp', name: 'created_at' }) + createdAt!: Date + + @Column({ type: 'timestamp', name: 'updated_at' }) + updatedAt!: Date +} diff --git a/pkg/admin/src/index.ts b/pkg/admin/src/index.ts index 687ddfb59..543c0d987 100644 --- a/pkg/admin/src/index.ts +++ b/pkg/admin/src/index.ts @@ -12,6 +12,7 @@ import { ContentDisplayReport, Subscription, Integration, + LibraryItem, } from './db' import { compare, hashSync } from 'bcryptjs' const readYamlFile = require('read-yaml-file') @@ -41,6 +42,7 @@ const ADMIN_USER_EMAIL = { resource: Group, options: { parent: { name: 'Users' } } }, { resource: Subscription, options: { parent: { name: 'Users' } } }, { resource: Integration, options: { parent: { name: 'Users' } } }, + { resource: LibraryItem, options: { parent: { name: 'Users' } } }, { resource: ContentDisplayReport, }, From ff2d7fd61704558b2b82049de1616428ad23a18e Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 23 Oct 2023 14:44:49 +0800 Subject: [PATCH 085/115] Extend base entity --- pkg/admin/src/db.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/admin/src/db.ts b/pkg/admin/src/db.ts index df4f0830c..e03de3976 100644 --- a/pkg/admin/src/db.ts +++ b/pkg/admin/src/db.ts @@ -311,7 +311,7 @@ export class Subscription extends BaseEntity { } @Entity({ name: 'library_item' }) -export class LibraryItem { +export class LibraryItem extends BaseEntity { @PrimaryGeneratedColumn('uuid') id!: string From 801c478381867ac6bb9b822e41c71383bcf45161 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 23 Oct 2023 14:55:50 +0800 Subject: [PATCH 086/115] Fix some columns --- pkg/admin/src/db.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/admin/src/db.ts b/pkg/admin/src/db.ts index e03de3976..ea423c2b6 100644 --- a/pkg/admin/src/db.ts +++ b/pkg/admin/src/db.ts @@ -319,7 +319,7 @@ export class LibraryItem extends BaseEntity { @ManyToOne(() => User, (user) => user.articles, { eager: true }) user!: User - @Column('text') + @Column({ type: 'text', name: 'original_url' }) originalUrl!: string @Column('text') @@ -331,12 +331,6 @@ export class LibraryItem extends BaseEntity { @Column('text', { nullable: true }) author?: string | null - @Column('text', { nullable: true }) - siteName?: string | null - - @Column('text') - itemType!: string - @Column('text', { nullable: true }) subscription?: string | null From 7fe4fec112bb70f70f14bb7dfa987ec7c5eb33c9 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 23 Oct 2023 15:07:30 +0800 Subject: [PATCH 087/115] Add upload_file --- pkg/admin/src/db.ts | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pkg/admin/src/db.ts b/pkg/admin/src/db.ts index ea423c2b6..73eb67bd9 100644 --- a/pkg/admin/src/db.ts +++ b/pkg/admin/src/db.ts @@ -334,6 +334,10 @@ export class LibraryItem extends BaseEntity { @Column('text', { nullable: true }) subscription?: string | null + @OneToOne(() => UploadFile, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'upload_file_id' }) + uploadFile?: UploadFile + @Column({ type: 'timestamp', name: 'saved_at' }) savedAt!: Date @@ -346,3 +350,31 @@ export class LibraryItem extends BaseEntity { @Column({ type: 'timestamp', name: 'updated_at' }) updatedAt!: Date } + +@Entity({ name: 'upload_files' }) +export class UploadFile extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id!: string + + @JoinColumn({ name: 'user_id' }) + @ManyToOne(() => User, (user) => user.articles, { eager: true }) + user!: User + + @Column('text') + url!: string + + @Column('text') + fileName!: string + + @Column('text') + contentType!: string + + @Column('text') + status!: string + + @Column({ type: 'timestamp', name: 'created_at' }) + createdAt!: Date + + @Column({ type: 'timestamp', name: 'updated_at' }) + updatedAt!: Date +} From a0d186df2f0822fdd4972e8e155dde9e3392e302 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Mon, 23 Oct 2023 15:16:15 +0800 Subject: [PATCH 088/115] Add entity mapping --- pkg/admin/src/db.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/admin/src/db.ts b/pkg/admin/src/db.ts index 73eb67bd9..15435dfe5 100644 --- a/pkg/admin/src/db.ts +++ b/pkg/admin/src/db.ts @@ -51,6 +51,7 @@ export const registerDatabase = async (secrets: any): Promise => { Integration, Subscription, LibraryItem, + UploadFile, ], }) From bbc46c63892ccba94b48e6f79c95adb5677ab735 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 15:43:19 +0800 Subject: [PATCH 089/115] fix: when source is rss-feed or csv-import, do not parse items in the backend --- packages/api/src/services/save_page.ts | 5 ++-- packages/api/test/resolvers/article.test.ts | 31 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index ad8d6d81c..3e82374d7 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -36,6 +36,7 @@ const FORCE_PUPPETEER_URLS = [ TWEET_URL_REGEX, /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/, ] +const ALREADY_PARSED_SOURCES = ['puppeteer-parse', 'csv-importer', 'rss-feeder'] const createSlug = (url: string, title?: Maybe | undefined) => { const { pathname } = new URL(url) @@ -52,7 +53,7 @@ const createSlug = (url: string, title?: Maybe | undefined) => { const shouldParseInBackend = (input: SavePageInput): boolean => { return ( - input.source !== 'puppeteer-parse' && + ALREADY_PARSED_SOURCES.indexOf(input.source) === -1 && FORCE_PUPPETEER_URLS.some((regex) => regex.test(input.url)) ) } @@ -99,7 +100,7 @@ export const savePage = async ( await createPageSaveRequest({ userId: user.id, url: itemToSave.originalUrl, - articleSavingRequestId: clientRequestId, + articleSavingRequestId: clientRequestId || undefined, state: input.state || undefined, labels: input.labels || undefined, }) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 7cb3cdcbe..17d7518d0 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -185,15 +185,17 @@ const savePageQuery = ( title: string, originalContent: string, state: ArticleSavingRequestStatus | null = null, - labels: string[] | null = null + labels: string[] | null = null, + clientRequestId = generateFakeUuid(), + source = 'puppeteer-parse' ) => { return ` mutation { savePage( input: { url: "${url}", - source: "test", - clientRequestId: "${generateFakeUuid()}", + source: "${source}", + clientRequestId: "${clientRequestId}", title: "${title}", originalContent: "${originalContent}" state: ${state} @@ -605,6 +607,29 @@ describe('Article API', () => { expect(savedItem?.labels?.map((l) => l.name)).to.eql(labels) }) }) + + context('when the source is rss-feeder', () => { + const source = 'rss-feeder' + const stub = sinon.stub(createTask, 'enqueueParseRequest') + + before(() => { + url = 'https://blog.omnivore.app/new-url-3' + }) + + after(async () => { + await deleteLibraryItemByUrl(url, user.id) + sinon.restore() + }) + + it('does not parse in the backend', async () => { + await graphqlRequest( + savePageQuery(url, title, originalContent, null, null, '', source), + authToken + ).expect(200) + + expect(stub).not.to.have.been.called + }) + }) }) describe('SaveUrl', () => { From b26e15c5d62ee076d8d9533fac9c0b9cdba1ac28 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 15:44:47 +0800 Subject: [PATCH 090/115] lint --- packages/api/src/resolvers/article/index.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 54b9fcfba..85a4bd4de 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -53,11 +53,9 @@ import { getInternalLabelWithColor } from '../../repository/label' import { libraryItemRepository } from '../../repository/library_item' import { userRepository } from '../../repository/user' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { findHighlightsByLibraryItemId } from '../../services/highlights' import { addLabelsToLibraryItem, findLabelsByIds, - findLabelsByLibraryItemId, findOrCreateLabels, saveLabelsInLibraryItem, } from '../../services/labels' @@ -70,7 +68,6 @@ import { updateLibraryItemReadingProgress, updateLibraryItems, } from '../../services/library_item' -import { findRecommendationsByLibraryItemId } from '../../services/recommendation' import { parsedContentToLibraryItem } from '../../services/save_page' import { findUploadFileById, @@ -84,14 +81,12 @@ import { cleanUrl, errorHandler, generateSlug, - isBase64Image, isParsingTimeout, libraryItemToArticle, libraryItemToSearchItem, titleForFilePath, userDataToUser, } from '../../utils/helpers' -import { createImageProxyUrl } from '../../utils/imageproxy' import { contentConverter, getDistillerResult, From 4d255a12a7ee267fa765dc42377ab12f0fcaeee0 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 15:47:21 +0800 Subject: [PATCH 091/115] add test cases --- packages/api/test/resolvers/article.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 17d7518d0..4f8a3c96e 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -608,12 +608,12 @@ describe('Article API', () => { }) }) - context('when the source is rss-feeder', () => { + context('when the source is rss-feeder and url is from youtube.com', () => { const source = 'rss-feeder' const stub = sinon.stub(createTask, 'enqueueParseRequest') before(() => { - url = 'https://blog.omnivore.app/new-url-3' + url = 'https://www.youtube.com/watch?v=123' }) after(async () => { From e94a61a9bcb76139b90ef706eda5798dd3562db3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 17:44:38 +0800 Subject: [PATCH 092/115] fix: library item id could be updated if a different client request id supplied in save page api payload --- packages/api/src/services/save_page.ts | 15 +++++++-- packages/api/test/resolvers/article.test.ts | 34 +++++++-------------- 2 files changed, 23 insertions(+), 26 deletions(-) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 3e82374d7..916d998c4 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -78,6 +78,7 @@ export const savePage = async ( let clientRequestId = input.clientRequestId const itemToSave = parsedContentToLibraryItem({ + itemId: clientRequestId, url: input.url, title: input.title, userId: user.id, @@ -119,6 +120,9 @@ export const savePage = async ( }) ) if (existingLibraryItem) { + clientRequestId = existingLibraryItem.id + slug = existingLibraryItem.slug + // we don't want to update an rss feed item if rss-feeder is tring to re-save it if (existingLibraryItem.subscription === input.rssFeedUrl) { return { @@ -127,11 +131,14 @@ export const savePage = async ( } } - clientRequestId = existingLibraryItem.id - slug = existingLibraryItem.slug + // update the item except for id and slug await updateLibraryItem( clientRequestId, - itemToSave as QueryDeepPartialEntity, + { + ...itemToSave, + id: undefined, + slug: undefined, + } as QueryDeepPartialEntity, user.id ) } else { @@ -256,5 +263,7 @@ export const parsedContentToLibraryItem = ({ wordCount: wordsCount(parsedContent?.textContent || ''), contentReader: contentReaderForLibraryItem(itemType, uploadFileId), subscription: rssFeedUrl, + archivedAt: + state === ArticleSavingRequestStatus.Archived ? new Date() : undefined, } } diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 4f8a3c96e..fbf3b2d86 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -16,7 +16,7 @@ import { PageType, SyncUpdatedItemEdge, UpdateReason, - UploadFileStatus, + UploadFileStatus } from '../../src/generated/graphql' import { getRepository } from '../../src/repository' import { createGroup, deleteGroup } from '../../src/services/groups' @@ -24,7 +24,7 @@ import { createHighlight } from '../../src/services/highlights' import { createLabel, deleteLabels, - saveLabelsInLibraryItem, + saveLabelsInLibraryItem } from '../../src/services/labels' import { createLibraryItem, @@ -35,7 +35,7 @@ import { deleteLibraryItemsByUserId, findLibraryItemById, findLibraryItemByUrl, - updateLibraryItem, + updateLibraryItem } from '../../src/services/library_item' import { deleteUser } from '../../src/services/user' import * as createTask from '../../src/utils/createTask' @@ -579,22 +579,26 @@ describe('Article API', () => { // Now save the link again, and ensure it is returned await graphqlRequest( - savePageQuery(url, title, originalContent), + savePageQuery(url, title, originalContent, null, null, generateFakeUuid()), authToken ).expect(200) allLinks = await graphqlRequest(searchQuery(''), authToken).expect(200) + expect(allLinks.body.data.search.edges[0].node.id).to.eq(justSavedId) expect(allLinks.body.data.search.edges[0].node.url).to.eq(url) }) }) - xcontext('when we also want to save labels and archives the item', () => { + context('when we also want to save labels and archives the item', () => { + before(() => { + url = 'https://blog.omnivore.app/new-url-2' + }) + after(async () => { - await deleteLibraryItemById(url, user.id) + await deleteLibraryItemByUrl(url, user.id) }) it('saves the labels and archives the item', async () => { - url = 'https://blog.omnivore.app/new-url-2' const state = ArticleSavingRequestStatus.Archived const labels = ['test name', 'test name 2'] await graphqlRequest( @@ -660,22 +664,6 @@ describe('Article API', () => { ) }) }) - - xcontext('when we save labels', () => { - it('saves the labels and archives the item', async () => { - url = 'https://blog.omnivore.app/new-url-2' - const state = ArticleSavingRequestStatus.Archived - const labels = ['test name', 'test name 2'] - await graphqlRequest( - saveUrlQuery(url, state, labels), - authToken - ).expect(200) - - const savedItem = await findLibraryItemByUrl(url, user.id) - expect(savedItem?.archivedAt).to.not.be.null - expect(savedItem?.labels?.map((l) => l.name)).to.eql(labels) - }) - }) }) describe('setBookmarkArticle', () => { From 27b352f9ae897df936725c98adc76310d701734c Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 19:07:47 +0800 Subject: [PATCH 093/115] fix: updates since api returns error for android client if invalid date is supplied --- packages/api/src/resolvers/article/index.ts | 8 +++++++- packages/api/test/resolvers/article.test.ts | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 85a4bd4de..f0e4cd327 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -46,6 +46,7 @@ import { TypeaheadSearchSuccess, UpdateReason, UpdatesSinceError, + UpdatesSinceErrorCode, UpdatesSinceSuccess, } from '../../generated/graphql' import { getColumns } from '../../repository' @@ -702,7 +703,12 @@ export const updatesSinceResolver = authorized< const startCursor = after || '' const size = first || 10 - const startDate = new Date(since) + let startDate = new Date(since) + if (isNaN(startDate.getTime())) { + // for android app compatibility + startDate = new Date(0) + } + const { libraryItems, count } = await searchLibraryItems( { from: Number(startCursor), diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index fbf3b2d86..4607044f4 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -1670,6 +1670,21 @@ describe('Article API', () => { UpdateReason.Deleted ) }) + + context('when since is -1000000000-01-01T00:00:00Z from android app', () => { + before(() => { + since = '-1000000000-01-01T00:00:00Z' + }) + + it('returns all', async () => { + const res = await graphqlRequest( + updatesSinceQuery(since), + authToken + ).expect(200) + + expect(res.body.data.updatesSince.edges.length).to.eql(5) + }) + }) }) describe('BulkAction API', () => { From 0c60706503887e1b96a003ef17fa089824778a58 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 19:34:41 +0800 Subject: [PATCH 094/115] fix: date filter in bulk action api --- packages/api/src/services/save_page.ts | 8 +++----- packages/api/src/utils/search.ts | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 916d998c4..68af11b2c 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -1,6 +1,7 @@ import { Readability } from '@omnivore/readability' import { DeepPartial } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' +import { Highlight } from '../entity/highlight' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { User } from '../entity/user' import { homePageURL } from '../env' @@ -166,12 +167,9 @@ export const savePage = async ( } if (parseResult.highlightData) { - const highlight = { - updatedAt: new Date(), - createdAt: new Date(), - userId: user.id, + const highlight: DeepPartial = { ...parseResult.highlightData, - type: HighlightType.Highlight, + user: { id: user.id }, } if (!(await createHighlight(highlight, clientRequestId, user.id))) { diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index c4799ba7b..96abd859b 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -249,13 +249,13 @@ const parseDateFilter = ( switch (field.toUpperCase()) { case 'PUBLISHED': - field = 'publishedAt' + field = 'published_at' break case 'SAVED': - field = 'savedAt' + field = 'saved_at' break case 'UPDATED': - field = 'updatedAt' + field = 'updated_at' } return { From f18ea7a7ce62d80166e265d214d38773c7ec55a3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 20:57:20 +0800 Subject: [PATCH 095/115] ignore position check if force = true in reading progress api payload --- packages/api/src/generated/graphql.ts | 1 + packages/api/src/generated/schema.graphql | 1 + packages/api/src/resolvers/article/index.ts | 21 +++++++++++ packages/api/src/schema.ts | 1 + packages/api/test/resolvers/article.test.ts | 41 +++++++++++++++++++-- 5 files changed, 61 insertions(+), 4 deletions(-) diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index 3bbc9a88d..da68dbe22 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2092,6 +2092,7 @@ export enum SaveArticleReadingProgressErrorCode { } export type SaveArticleReadingProgressInput = { + force?: InputMaybe; id: Scalars['ID']; readingProgressAnchorIndex?: InputMaybe; readingProgressPercent: Scalars['Float']; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index e5192b308..dec3e34f6 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1571,6 +1571,7 @@ enum SaveArticleReadingProgressErrorCode { } input SaveArticleReadingProgressInput { + force: Boolean id: ID! readingProgressAnchorIndex: Int readingProgressPercent: Float! diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 85a4bd4de..086b752b1 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -563,6 +563,7 @@ export const saveArticleReadingProgressResolver = authorized< readingProgressPercent, readingProgressAnchorIndex, readingProgressTopPercent, + force, }, }, { log, pubsub, uid } @@ -578,6 +579,26 @@ export const saveArticleReadingProgressResolver = authorized< return { errorCodes: [SaveArticleReadingProgressErrorCode.BadData] } } try { + if (force) { + // update reading progress without checking the current value + const updatedItem = await updateLibraryItem( + id, + { + readingProgressBottomPercent: readingProgressPercent, + readingProgressTopPercent: readingProgressTopPercent ?? undefined, + readingProgressHighestReadAnchor: + readingProgressAnchorIndex ?? undefined, + }, + uid, + pubsub + ) + + return { + updatedArticle: libraryItemToArticle(updatedItem), + } + } + + // update reading progress only if the current value is lower const updatedItem = await updateLibraryItemReadingProgress( id, uid, diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index de3f5209c..ff9d75664 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -637,6 +637,7 @@ const schema = gql` readingProgressTopPercent: Float readingProgressPercent: Float! readingProgressAnchorIndex: Int + force: Boolean } enum SaveArticleReadingProgressErrorCode { NOT_FOUND diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 4f8a3c96e..6c1c51446 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -295,16 +295,18 @@ const setBookmarkQuery = (articleId: string, bookmark: boolean) => { const saveArticleReadingProgressQuery = ( articleId: string, progress: number, - topPercent: number | null = null + topPercent: number | null = null, + force: boolean | null = null ) => { return ` mutation { saveArticleReadingProgress( input: { id: "${articleId}", - readingProgressPercent: ${progress} - readingProgressAnchorIndex: 0 - readingProgressTopPercent: ${topPercent} + readingProgressPercent: ${progress}, + readingProgressAnchorIndex: 0, + readingProgressTopPercent: ${topPercent}, + force: ${force} } ) { ... on SaveArticleReadingProgressSuccess { @@ -784,6 +786,37 @@ describe('Article API', () => { 'BAD_DATA', ]) }) + + context('when force is true', () => { + before(async () => { + itemId = (await createLibraryItem({ + user: { id: user.id }, + originalUrl: 'https://blog.omnivore.app/setBookmarkArticle', + slug: 'test-with-omnivore', + readableContent: '

                    test

                    ', + title: 'test title', + readingProgressBottomPercent: 100, + readingProgressTopPercent: 80, + }, user.id)).id + }) + + after(async () => { + await deleteLibraryItemById(itemId, user.id) + }) + + it('ignore position check if force is true', async () => { + query = saveArticleReadingProgressQuery(itemId, 20, 10, true) + const res = await graphqlRequest(query, authToken).expect(200) + expect( + res.body.data.saveArticleReadingProgress.updatedArticle + .readingProgressPercent + ).to.eql(20) + expect( + res.body.data.saveArticleReadingProgress.updatedArticle + .readingProgressTopPercent + ).to.eql(10) + }) + }) }) describe('SaveFile', () => { From e673db423b8503cf8b078d8ecbf6f8c9f99c41b8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 21:11:37 +0800 Subject: [PATCH 096/115] limit max content size to 10MB --- packages/api/src/schema.ts | 6 ++++-- packages/api/src/services/library_item.ts | 17 ++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index de3f5209c..9bfe8406b 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -543,7 +543,8 @@ const schema = gql` title: String! byline: String dir: String - content: String! + # max length of 10MB + content: String! @sanitize(maxLength: 10485760) textContent: String! length: Int! excerpt: String! @@ -559,7 +560,8 @@ const schema = gql` source: String! clientRequestId: ID! title: String - originalContent: String! + # max length of 10MB + originalContent: String! @sanitize(maxLength: 10485760) parseResult: ParseResult state: ArticleSavingRequestStatus labels: [CreateLabelInput!] diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 8dab74fc5..e662ecb69 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -431,7 +431,13 @@ export const updateLibraryItem = async ( await pubsub.entityUpdated>( EntityType.PAGE, - { ...libraryItem, id }, + { + ...libraryItem, + id, + // don't send original content and readable content + originalContent: undefined, + readableContent: undefined, + }, userId ) @@ -535,9 +541,14 @@ export const createLibraryItem = async ( userId ) - await pubsub.entityCreated( + await pubsub.entityCreated>( EntityType.PAGE, - newLibraryItem, + { + ...newLibraryItem, + // don't send original content and readable content + originalContent: undefined, + readableContent: undefined, + }, userId ) From f46bb616fa78b062d507847b057bd5418ad5a3eb Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 21:14:23 +0800 Subject: [PATCH 097/115] update read_at to now --- packages/api/src/resolvers/article/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 086b752b1..e9802667e 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -588,6 +588,7 @@ export const saveArticleReadingProgressResolver = authorized< readingProgressTopPercent: readingProgressTopPercent ?? undefined, readingProgressHighestReadAnchor: readingProgressAnchorIndex ?? undefined, + readAt: new Date(), }, uid, pubsub From 79a647deb45ae50593c972d81d85ad3ef0004b75 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 21:27:16 +0800 Subject: [PATCH 098/115] do not publish content and original html in the queue --- packages/api/src/resolvers/article/index.ts | 2 +- packages/api/src/schema.ts | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index f0e4cd327..c45b4b67d 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -623,7 +623,7 @@ export const searchResolver = authorized< size: first + 1, // fetch one more item to get next cursor sort: searchQuery.sort, includePending: true, - includeContent: params.includeContent || false, + includeContent: !!params.includeContent, ...searchQuery, }, uid diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 9bfe8406b..de3f5209c 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -543,8 +543,7 @@ const schema = gql` title: String! byline: String dir: String - # max length of 10MB - content: String! @sanitize(maxLength: 10485760) + content: String! textContent: String! length: Int! excerpt: String! @@ -560,8 +559,7 @@ const schema = gql` source: String! clientRequestId: ID! title: String - # max length of 10MB - originalContent: String! @sanitize(maxLength: 10485760) + originalContent: String! parseResult: ParseResult state: ArticleSavingRequestStatus labels: [CreateLabelInput!] From 3f14ac4f9b0da3ddafe01761a0b5daeba91b6845 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 21:31:38 +0800 Subject: [PATCH 099/115] skip publishing event for importing --- packages/api/src/services/library_item.ts | 7 ++++++- packages/api/src/services/save_page.ts | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index e662ecb69..7fec735e0 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -527,7 +527,8 @@ export const createLibraryItems = async ( export const createLibraryItem = async ( libraryItem: DeepPartial, userId: string, - pubsub = createPubSubClient() + pubsub = createPubSubClient(), + skipPubSub = false ): Promise => { const newLibraryItem = await authTrx( async (tx) => @@ -541,6 +542,10 @@ export const createLibraryItem = async ( userId ) + if (skipPubSub) { + return newLibraryItem + } + await pubsub.entityCreated>( EntityType.PAGE, { diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 68af11b2c..e14ca4850 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -7,7 +7,6 @@ import { User } from '../entity/user' import { homePageURL } from '../env' import { ArticleSavingRequestStatus, - HighlightType, Maybe, PreparedDocumentInput, SaveErrorCode, @@ -144,7 +143,12 @@ export const savePage = async ( ) } else { // do not publish a pubsub event if the item is imported - const newItem = await createLibraryItem(itemToSave, user.id) + const newItem = await createLibraryItem( + itemToSave, + user.id, + undefined, + isImported + ) clientRequestId = newItem.id } From f497c0ac5ba917e0c0e38670c193b919510bbe06 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 21:43:01 +0800 Subject: [PATCH 100/115] add test cases --- packages/api/test/resolvers/article.test.ts | 60 +++++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index 4607044f4..c50466e13 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -1741,18 +1741,56 @@ describe('Article API', () => { }) }) - context('when action is Archive', () => { - it('archives all items', async () => { - const res = await graphqlRequest( - bulkActionQuery(BulkActionType.Archive), - authToken - ).expect(200) - expect(res.body.data.bulkAction.success).to.be.true + context( + 'when action is Archive and query is published:*..2023-10-01', + () => { + let items: LibraryItem[] = [] - const items = await graphqlRequest(searchQuery(), authToken).expect(200) - expect(items.body.data.search.pageInfo.totalCount).to.eql(0) - }) - }) + before(async () => { + items = await createLibraryItems( + [ + { + user, + title: 'test item', + readableContent: '

                    test

                    ', + slug: 'test-item', + originalUrl: `https://blog.omnivore.app/p/bulk-action-archive`, + publishedAt: new Date('2023-10-01'), + }, + { + user, + title: 'test item 2', + readableContent: '

                    test

                    ', + slug: 'test-item-2', + originalUrl: `https://blog.omnivore.app/p/bulk-action-archive-2`, + publishedAt: new Date('2023-10-02'), + }, + ], + user.id + ) + }) + + after(async () => { + // Delete all items + await deleteLibraryItems(items, user.id) + }) + + it('archives old items', async () => { + const res = await graphqlRequest( + bulkActionQuery(BulkActionType.Archive, 'published:*..2023-10-01'), + authToken + ).expect(200) + expect(res.body.data.bulkAction.success).to.be.true + + const response = await graphqlRequest( + searchQuery('in:archive'), + authToken + ).expect(200) + expect(response.body.data.search.pageInfo.totalCount).to.eql(1) + expect(response.body.data.search.edges[0].node.id).to.eql(items[0].id) + }) + } + ) context('when action is Delete', () => { it('deletes all items', async () => { From 41866d5cb7213f3bd1c331be7c145f9764b9e517 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 23 Oct 2023 22:02:43 +0800 Subject: [PATCH 101/115] cont --- packages/api/src/services/save_page.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index e14ca4850..80206277b 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -174,6 +174,7 @@ export const savePage = async ( const highlight: DeepPartial = { ...parseResult.highlightData, user: { id: user.id }, + libraryItem: { id: clientRequestId }, } if (!(await createHighlight(highlight, clientRequestId, user.id))) { From 3c75a23e75eb68a6d6e24bd8f78537f7ceb1bc82 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 24 Oct 2023 12:02:58 +0800 Subject: [PATCH 102/115] fix: showing original url instead of signed url for pdf files in search api response --- packages/api/src/entity/library_item.ts | 4 ++++ packages/api/src/resolvers/function_resolvers.ts | 2 +- packages/api/src/utils/helpers.ts | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 416c65b68..31383da62 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -129,6 +129,10 @@ export class LibraryItem { @JoinColumn({ name: 'upload_file_id' }) uploadFile?: UploadFile + // get upload_file_id without loading the uploadFile entity + @Column('text', { nullable: true }) + uploadFileId?: string + @Column('enum', { enum: ContentReaderType, default: ContentReaderType.WEB }) contentReader!: ContentReaderType diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index bc58b54bf..ca145815e 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -465,7 +465,7 @@ export const functionResolvers = { async url(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) { if ( (item.pageType == PageType.File || item.pageType == PageType.Book) && - ctx.uid && + ctx.claims && item.uploadFileId ) { const upload = await findUploadFileById(item.uploadFileId) diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index dadcfe171..bdfe443c1 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -268,7 +268,6 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ ), image: item.thumbnail, highlights: item.highlights?.map(highlightDataToHighlight), - uploadFileId: item.uploadFile?.id, wordsCount: item.wordCount, }) From d7bd5c54a379d4e348450465ac2a5591501cef2a Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 11:43:05 +0800 Subject: [PATCH 103/115] iOS: fix for PDF downloads, always use the download signed URL, not the original URL --- .../Sources/App/PDFSupport/PDFViewerViewModel.swift | 2 +- apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift | 2 +- .../CoreDataModel.xcdatamodel/contents | 3 ++- apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift | 2 ++ apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift | 4 ++-- .../Sources/Services/DataService/ContentLoading.swift | 2 +- .../Sources/Services/DataService/Public/PDFLoading.swift | 4 ++-- .../Services/DataService/Queries/ArticleContentQuery.swift | 1 + .../Services/DataService/Queries/LinkedItemNetworkQuery.swift | 2 ++ .../Sources/Services/InternalModels/InternalLinkedItem.swift | 3 +++ 10 files changed, 17 insertions(+), 8 deletions(-) diff --git a/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift b/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift index 529fe192c..4643ef46d 100644 --- a/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift +++ b/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift @@ -96,7 +96,7 @@ final class PDFViewerViewModel: ObservableObject { } } - return try await dataService.loadPDFData(slug: pdfItem.slug, pageURLString: pdfItem.originalArticleURL) + return try await dataService.loadPDFData(slug: pdfItem.slug, downloadURL: pdfItem.downloadURL) } catch { print("error downloading PDF", error) return nil diff --git a/apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift b/apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift index 70ef246e1..1b7e4aa4a 100644 --- a/apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift @@ -46,7 +46,7 @@ import Views private func trackReadEvent() { guard let itemID = item?.unwrappedID ?? pdfItem?.itemID else { return } guard let slug = item?.unwrappedSlug ?? pdfItem?.slug else { return } - guard let originalArticleURL = item?.unwrappedPageURLString ?? pdfItem?.originalArticleURL else { return } + guard let originalArticleURL = item?.unwrappedPageURLString ?? pdfItem?.downloadURL else { return } EventTracker.track( .linkRead( diff --git a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents index 2fd575363..0351d7ee9 100644 --- a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents +++ b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents @@ -1,5 +1,5 @@ - + @@ -32,6 +32,7 @@ + diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift index 92d4eda80..b7c77bc5b 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift @@ -56,12 +56,14 @@ public struct JSONArticle: Decodable { public let isArchived: Bool public let language: String? public let wordsCount: Int? + public let downloadURL: String } public extension LinkedItem { var unwrappedID: String { id ?? "" } var unwrappedSlug: String { slug ?? "" } var unwrappedTitle: String { title ?? "" } + var unwrappedDownloadURLString: String { downloadURL ?? "" } var unwrappedPageURLString: String { pageURLString ?? "" } var unwrappedSavedAt: Date { savedAt ?? Date() } var unwrappedCreatedAt: Date { createdAt ?? Date() } diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift b/apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift index 43c2eb672..224e60901 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift @@ -14,7 +14,7 @@ public struct PDFItem { public let readingProgressAnchor: Int public let isArchived: Bool public let isRead: Bool - public let originalArticleURL: String + public let downloadURL: String public let highlights: [Highlight] public static func make(item: LinkedItem) -> PDFItem? { @@ -32,7 +32,7 @@ public struct PDFItem { readingProgressAnchor: Int(item.readingProgressAnchor), isArchived: item.isArchived, isRead: item.isRead, - originalArticleURL: item.unwrappedPageURLString, + downloadURL: item.unwrappedPageURLString, highlights: item.highlights.asArray(of: Highlight.self) ) } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift index d276c40e1..e96469e3a 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift @@ -166,7 +166,7 @@ extension DataService { } if articleProps.item.isPDF, needsPDFDownload { - _ = try await loadPDFData(slug: articleProps.item.slug, pageURLString: articleProps.item.pageURLString) + _ = try await loadPDFData(slug: articleProps.item.slug, downloadURL: articleProps.item.downloadURL) } try await backgroundContext.perform { [weak self] in diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Public/PDFLoading.swift b/apple/OmnivoreKit/Sources/Services/DataService/Public/PDFLoading.swift index 0f4315844..8b6520640 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Public/PDFLoading.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Public/PDFLoading.swift @@ -4,8 +4,8 @@ import Models import Utils public extension DataService { - func loadPDFData(slug: String, pageURLString: String) async throws -> URL? { - guard let url = URL(string: pageURLString) else { + func loadPDFData(slug: String, downloadURL: String) async throws -> URL? { + guard let url = URL(string: downloadURL) else { throw BasicError.message(messageText: "No PDF URL found") } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift index 56e7a77ea..a1427b35a 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift @@ -45,6 +45,7 @@ extension DataService { originalHtml: nil, language: try $0.language(), wordsCount: try $0.wordsCount(), + downloadURL: try $0.url(), recommendations: try $0.recommendations(selection: recommendationSelection.list.nullable) ?? [], labels: try $0.labels(selection: feedItemLabelSelection.list.nullable) ?? [] ), diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift index 7f897bc04..548ba02ea 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift @@ -277,6 +277,7 @@ private let libraryArticleSelection = Selection.Article { originalHtml: nil, language: try $0.language(), wordsCount: try $0.wordsCount(), + downloadURL: try $0.url(), recommendations: try $0.recommendations(selection: recommendationSelection.list.nullable) ?? [], labels: try $0.labels(selection: feedItemLabelSelection.list.nullable) ?? [] ) @@ -316,6 +317,7 @@ private let searchItemSelection = Selection.SearchItem { originalHtml: nil, language: try $0.language(), wordsCount: try $0.wordsCount(), + downloadURL: try $0.url(), recommendations: try $0.recommendations(selection: recommendationSelection.list.nullable) ?? [], labels: try $0.labels(selection: feedItemLabelSelection.list.nullable) ?? [] ) diff --git a/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift b/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift index 43a7dcce9..3031cd1ef 100644 --- a/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift +++ b/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift @@ -27,6 +27,7 @@ struct InternalLinkedItem { let originalHtml: String? let language: String? let wordsCount: Int? + let downloadURL: String let recommendations: [InternalRecommendation] var labels: [InternalLinkedItemLabel] @@ -65,6 +66,7 @@ struct InternalLinkedItem { linkedItem.originalHtml = originalHtml linkedItem.language = language linkedItem.wordsCount = Int64(wordsCount ?? 0) + linkedItem.downloadURL = downloadURL // Remove existing labels in case a label had been deleted if let existingLabels = linkedItem.labels { @@ -146,6 +148,7 @@ extension JSONArticle { originalHtml: nil, language: language, wordsCount: wordsCount, + downloadURL: downloadURL, recommendations: [], labels: [] ) From 9a1ccd46bcaeb2450778556024bcaeafca9fb907 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 12:11:28 +0800 Subject: [PATCH 104/115] Bump version, attempt to refetch PDFs if download signed URL has expired --- .../App/PDFSupport/PDFViewerViewModel.swift | 13 +++++++++- .../Models/DataModels/ArticleContent.swift | 5 +++- .../Services/DataService/ContentLoading.swift | 6 +++-- packages/api/src/server.ts | 25 +++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift b/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift index 4643ef46d..af5f05915 100644 --- a/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift +++ b/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift @@ -96,7 +96,18 @@ final class PDFViewerViewModel: ObservableObject { } } - return try await dataService.loadPDFData(slug: pdfItem.slug, downloadURL: pdfItem.downloadURL) + if let result = try? await dataService.loadPDFData(slug: pdfItem.slug, downloadURL: pdfItem.downloadURL) { + return result + } + + // Downloading failed, try to get the article again, and then download + if let content = try? await dataService.loadArticleContentWithRetries(itemID: pdfItem.itemID, username: "me") { + // refetched the content, now try one more time then throw + if let result = try await dataService.loadPDFData(slug: pdfItem.slug, downloadURL: content.downloadURL) { + return result + } + } + return nil } catch { print("error downloading PDF", error) return nil diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/ArticleContent.swift b/apple/OmnivoreKit/Sources/Models/DataModels/ArticleContent.swift index 2a453a9ac..4ce4eb657 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/ArticleContent.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/ArticleContent.swift @@ -16,19 +16,22 @@ public struct ArticleContent { public let highlightsJSONString: String public let contentStatus: ArticleContentStatus public let objectID: NSManagedObjectID? + public let downloadURL: String public init( title: String, htmlContent: String, highlightsJSONString: String, contentStatus: ArticleContentStatus, - objectID: NSManagedObjectID? + objectID: NSManagedObjectID?, + downloadURL: String ) { self.title = title self.htmlContent = htmlContent self.highlightsJSONString = highlightsJSONString self.contentStatus = contentStatus self.objectID = objectID + self.downloadURL = downloadURL } } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift index e96469e3a..a5427549b 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift @@ -63,7 +63,8 @@ extension DataService { htmlContent: fetchResult.htmlContent, highlightsJSONString: fetchResult.highlights.asJSONString, contentStatus: fetchResult.item.isPDF ? .succeeded : fetchResult.item.state, - objectID: objectID + objectID: objectID, + downloadURL: fetchResult.item.downloadURL ) } @@ -91,7 +92,8 @@ extension DataService { .filter { $0.serverSyncStatus != ServerSyncStatus.needsDeletion.rawValue } .map { InternalHighlight.make(from: $0) }.asJSONString, contentStatus: .succeeded, - objectID: linkedItem.objectID + objectID: linkedItem.objectID, + downloadURL: linkedItem.downloadURL ?? "" ) } } diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index b40a82bb8..5c6c27d87 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -38,6 +38,7 @@ import { sentryConfig } from './sentry' import { getClaimsByToken, getTokenByRequest } from './utils/auth' import { corsConfig } from './utils/corsConfig' import { buildLogger, buildLoggerTransport } from './utils/logger' +import { FirefishClient, createFirefishUserClient } from './activitypub' const PORT = process.env.PORT || 4000 @@ -178,4 +179,28 @@ const main = async (): Promise => { // only call main if the file was called from the CLI and wasn't required from another module if (require.main === module) { main() + ;(async () => { + // console.log( + // 'creating user with firefish token: ', + // process.env.FIREFISH_TOKEN + // ) + // const client = new FirefishClient( + // 'http://localhost:8000/api', + // process.env.FIREFISH_TOKEN || 'firefish-token', + // 'abc123' + // ) + // // const token = await client.createUserActor( + // // 'a03a7396-909b-11ed-9075-c3f3cf07eed9', + // // 'jacksonh@gmail.com', + // // 'jacksonharper' + // // ) + // // console.log('user token: ', token) + // // if (token) { + // // const secret = client.createAppForActor(token) + // // console.log('created secretL: ', secret) + // // } + // const appClient = createFirefishUserClient('UEpw5pStVG5hFhrX') + // appClient.setupUserActor('a03a7396-909b-11ed-9075-c3f3cf07eed9') + // appClient.getUserInfo() + })() } From 6bce5b5e19f55926aee67d977a3bae084208c72c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 12:36:54 +0800 Subject: [PATCH 105/115] Improve PDF downloading on Android --- android/Omnivore/app/build.gradle | 4 +- .../omnivore/networking/SavedItemQuery.kt | 22 +++++- .../omnivore/persistence/AppDatabase.kt | 2 +- .../persistence/entities/SavedItem.kt | 3 +- .../omnivore/ui/reader/PDFReaderViewModel.kt | 77 +++++++++++++------ .../omnivore/ui/reader/WebReaderContent.kt | 2 +- 6 files changed, 80 insertions(+), 30 deletions(-) diff --git a/android/Omnivore/app/build.gradle b/android/Omnivore/app/build.gradle index 1041a5ec9..beeda4ff1 100644 --- a/android/Omnivore/app/build.gradle +++ b/android/Omnivore/app/build.gradle @@ -17,8 +17,8 @@ android { applicationId "app.omnivore.omnivore" minSdk 26 targetSdk 33 - versionCode 118 - versionName "0.0.118" + versionCode 122 + versionName "0.0.122" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" vectorDrawables { diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemQuery.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemQuery.kt index 2fd8e9298..cd2f50e35 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemQuery.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemQuery.kt @@ -1,9 +1,15 @@ package app.omnivore.omnivore.networking +import android.util.Log import app.omnivore.omnivore.graphql.generated.GetArticleQuery +import app.omnivore.omnivore.graphql.generated.type.ContentReader import app.omnivore.omnivore.persistence.entities.SavedItem import app.omnivore.omnivore.persistence.entities.SavedItemLabel import app.omnivore.omnivore.persistence.entities.Highlight +import java.io.File +import java.net.URL +import java.nio.file.Files +import java.nio.file.StandardCopyOption data class SavedItemQueryResponse( val item: SavedItem?, @@ -60,7 +66,18 @@ suspend fun Networker.savedItem(slug: String): SavedItemQueryResponse { ) } - // TODO: handle errors + var localPDFPath: String? = null + if (article.articleFields.contentReader == ContentReader.PDF) { + // download the PDF and save it locally + // article.articleFields.url + + val localFile = File.createTempFile("pdf-" + article.articleFields.id, ".pdf", ) + val url = URL(article.articleFields.url) + Log.d("pdf", "creating local file: $localFile") + + url.openStream().use { Files.copy(it, localFile.toPath(), StandardCopyOption.REPLACE_EXISTING) } + localPDFPath = localFile.toPath().toString() + } val savedItem = SavedItem( savedItemId = article.articleFields.id, @@ -82,7 +99,8 @@ suspend fun Networker.savedItem(slug: String): SavedItemQueryResponse { isArchived = article.articleFields.isArchived, contentReader = article.articleFields.contentReader.rawValue, content = article.articleFields.content, - wordsCount = article.articleFields.wordsCount + wordsCount = article.articleFields.wordsCount, + localPDFPath = localPDFPath ) return SavedItemQueryResponse(item = savedItem, highlights, labels = savedItemLabels, state = article.articleFields.state?.rawValue ?: "") diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/AppDatabase.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/AppDatabase.kt index 07202993f..bf98747fd 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/AppDatabase.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/AppDatabase.kt @@ -13,7 +13,7 @@ import app.omnivore.omnivore.persistence.entities.* SavedItemAndSavedItemLabelCrossRef::class, SavedItemAndHighlightCrossRef::class ], - version = 11 + version = 12 ) abstract class AppDatabase : RoomDatabase() { abstract fun viewerDao(): ViewerDao diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt index 863ebe7ae..7dbd1dfd0 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt @@ -40,7 +40,8 @@ data class SavedItem( @ColumnInfo(typeAffinity = ColumnInfo.BLOB) val pdfData: ByteArray? = null, var serverSyncStatus: Int = 0, val tempPDFURL: String? = null, - val wordsCount: Int? = null + val wordsCount: Int? = null, + val localPDFPath: String? = null // hasMany highlights // hasMany labels diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/PDFReaderViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/PDFReaderViewModel.kt index 056fa559c..e674b89ac 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/PDFReaderViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/PDFReaderViewModel.kt @@ -7,7 +7,7 @@ import androidx.lifecycle.MutableLiveData import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import app.omnivore.omnivore.DatastoreRepository -import app.omnivore.omnivore.EventTracker +import app.omnivore.omnivore.dataService.DataService import app.omnivore.omnivore.graphql.generated.type.CreateHighlightInput import app.omnivore.omnivore.graphql.generated.type.MergeHighlightInput import app.omnivore.omnivore.graphql.generated.type.UpdateHighlightInput @@ -21,11 +21,16 @@ import com.pspdfkit.document.download.DownloadJob import com.pspdfkit.document.download.DownloadRequest import com.pspdfkit.document.download.Progress import dagger.hilt.android.lifecycle.HiltViewModel +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import org.json.JSONObject import java.io.File import java.lang.Double.max import java.lang.Double.min +import java.lang.Exception +import java.net.URLEncoder +import java.nio.file.FileSystem import java.util.* import javax.inject.Inject @@ -38,8 +43,8 @@ data class PDFReaderParams( @HiltViewModel class PDFReaderViewModel @Inject constructor( private val datastoreRepo: DatastoreRepository, - private val networker: Networker, - private val eventTracker: EventTracker, + private val dataService: DataService, + private val networker: Networker ): ViewModel() { var annotationUnderNoteEdit: Annotation? = null val pdfReaderParamsLiveData = MutableLiveData(null) @@ -47,21 +52,51 @@ class PDFReaderViewModel @Inject constructor( fun loadItem(slug: String, context: Context) { viewModelScope.launch { + loadItemFromDB(slug) + loadItemFromNetwork(slug, context) + } + } + + private suspend fun loadItemFromDB(slug: String) { + withContext(Dispatchers.IO) { + val persistedItem = dataService.db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug) + persistedItem?.let { persistedItem -> + persistedItem?.savedItem?.localPDF?.let { localPDF -> + val localFile = File(localPDF) + + if (localFile.exists()) { + val articleContent = ArticleContent( + title = persistedItem.savedItem.title, + htmlContent = "", + highlights = persistedItem.highlights, + contentStatus = "SUCCEEDED", + objectID = "", + labelsJSONString = Gson().toJson(persistedItem.labels) + ) + + pdfReaderParamsLiveData.postValue( + PDFReaderParams( + persistedItem.savedItem, + articleContent, + Uri.fromFile(localFile) + ) + ) + } + } + } + } + } + + private suspend fun loadItemFromNetwork(slug: String, context: Context) { + withContext(Dispatchers.IO) { val articleQueryResult = networker.savedItem(slug) - - val article = articleQueryResult.item ?: return@launch - + val article = articleQueryResult.item ?: return@withContext val request = DownloadRequest.Builder(context) .uri(article.pageURLString) .build() val job = DownloadJob.startDownload(request) - job.setProgressListener(object : DownloadJob.ProgressListenerAdapter() { - override fun onProgress(progress: Progress) { -// progressBar.setProgress((100f * progress.bytesReceived / progress.totalBytes).toInt()) - } - override fun onComplete(output: File) { val articleContent = ArticleContent( title = article.title, @@ -72,22 +107,18 @@ class PDFReaderViewModel @Inject constructor( labelsJSONString = Gson().toJson(articleQueryResult.labels) ) - val pdfReaderParams = PDFReaderParams(article, articleContent, Uri.fromFile(output)) - - eventTracker.track("link_read", - com.posthog.android.Properties() - .putValue("linkID", pdfReaderParams.item.savedItemId) - .putValue("slug", pdfReaderParams.item.slug) - .putValue("originalArticleURL", pdfReaderParams.item.pageURLString) - .putValue("loaded_from", "network") - ) - currentReadingProgress = article.readingProgress - pdfReaderParamsLiveData.postValue(pdfReaderParams) + pdfReaderParamsLiveData.postValue( + PDFReaderParams( + article, + articleContent, + Uri.fromFile(output) + ) + ) } override fun onError(exception: Throwable) { -// handleDownloadError(exception) +// handleDownloadError(exception) } }) } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderContent.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderContent.kt index 571e423d7..e04f13591 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderContent.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderContent.kt @@ -91,7 +91,7 @@ data class WebReaderContent( url: `${item.pageURLString}`, title: `${articleContent.title.replace("`", "\\`")}`, content: document.getElementById('_omnivore-htmlContent').innerHTML, - originalArticleUrl: "${item.pageURLString}", + originalArticleUrl: "${item.publisherURLString}", contentReader: "WEB", readingProgressPercent: ${item.readingProgress}, readingProgressAnchorIndex: ${item.readingProgressAnchor}, From 2d47198a3f75ff474ab6485be84aef5e0137db96 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 12:58:01 +0800 Subject: [PATCH 106/115] Make private in package.json again to fix build --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index b02a1da45..55b813a4b 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "omnivore-app", "version": "1.0.0", "description": "", - "private": false, + "private": true, "workspaces": [ "packages/*" ], @@ -40,4 +40,4 @@ "yarn": "1.22.19" }, "dependencies": {} -} +} \ No newline at end of file From 056bf9ebf198d85a1dacfdcc65fa429ea01f24a1 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 24 Oct 2023 13:12:59 +0800 Subject: [PATCH 107/115] fix update sql --- packages/api/src/entity/library_item.ts | 2 +- packages/api/src/services/save_page.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 31383da62..07f4ccd05 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -129,7 +129,7 @@ export class LibraryItem { @JoinColumn({ name: 'upload_file_id' }) uploadFile?: UploadFile - // get upload_file_id without loading the uploadFile entity + // get upload_file_id without joining relations @Column('text', { nullable: true }) uploadFileId?: string diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index 3e82374d7..fa9110565 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -242,7 +242,7 @@ export const parsedContentToLibraryItem = ({ publishedAt: validatedDate( publishedAt || parsedContent?.publishedDate || undefined ), - uploadFile: { id: uploadFileId ?? undefined }, + uploadFileId: uploadFileId || undefined, readingProgressTopPercent: 0, readingProgressHighestReadAnchor: 0, state: state From f1b7ac9b21d5a95a4293606f02bf9c39d987e0f5 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 13:13:18 +0800 Subject: [PATCH 108/115] Revert ap import from another branch --- packages/api/src/server.ts | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 5c6c27d87..b40a82bb8 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -38,7 +38,6 @@ import { sentryConfig } from './sentry' import { getClaimsByToken, getTokenByRequest } from './utils/auth' import { corsConfig } from './utils/corsConfig' import { buildLogger, buildLoggerTransport } from './utils/logger' -import { FirefishClient, createFirefishUserClient } from './activitypub' const PORT = process.env.PORT || 4000 @@ -179,28 +178,4 @@ const main = async (): Promise => { // only call main if the file was called from the CLI and wasn't required from another module if (require.main === module) { main() - ;(async () => { - // console.log( - // 'creating user with firefish token: ', - // process.env.FIREFISH_TOKEN - // ) - // const client = new FirefishClient( - // 'http://localhost:8000/api', - // process.env.FIREFISH_TOKEN || 'firefish-token', - // 'abc123' - // ) - // // const token = await client.createUserActor( - // // 'a03a7396-909b-11ed-9075-c3f3cf07eed9', - // // 'jacksonh@gmail.com', - // // 'jacksonharper' - // // ) - // // console.log('user token: ', token) - // // if (token) { - // // const secret = client.createAppForActor(token) - // // console.log('created secretL: ', secret) - // // } - // const appClient = createFirefishUserClient('UEpw5pStVG5hFhrX') - // appClient.setupUserActor('a03a7396-909b-11ed-9075-c3f3cf07eed9') - // appClient.getUserInfo() - })() } From a5d03e01dbc140ab5173122eca87db483715f8b1 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 17:03:36 +0800 Subject: [PATCH 109/115] Label flair for the web --- .../LibraryCards/LibraryCardStyles.tsx | 41 +++++++++++++++++-- .../patterns/LibraryCards/LibraryGridCard.tsx | 12 ++++-- .../patterns/LibraryCards/LibraryListCard.tsx | 12 ++++-- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx b/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx index 52e4ddd43..e7fd81c5a 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx @@ -2,7 +2,13 @@ import dayjs from 'dayjs' import relativeTime from 'dayjs/plugin/relativeTime' import { ChangeEvent, useMemo } from 'react' import { LibraryItemNode } from '../../../lib/networking/queries/useGetLibraryItemsQuery' -import { Box, SpanBox, VStack } from '../../elements/LayoutPrimitives' +import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' +import { RecommendedFlairIcon } from '../../elements/icons/RecommendedFlairIcon' +import { PinnedFlairIcon } from '../../elements/icons/PinnedFlairIcon' +import { FavoriteFlairIcon } from '../../elements/icons/FavoriteFlairIcon' +import { NewsletterFlairIcon } from '../../elements/icons/NewsletterFlairIcon' +import { FeedFlairIcon } from '../../elements/icons/FeedFlairIcon' +import { Label } from '../../../lib/networking/fragments/labelFragment' dayjs.extend(relativeTime) @@ -83,6 +89,32 @@ const shouldHideUrl = (url: string): boolean => { return false } +export const FLAIR_ICON_NAMES = [ + 'favorite', + 'pinned', + 'recommended', + 'newsletter', + 'feed', + 'rss', +] + +const flairIconForLabel = (label: Label): JSX.Element | undefined => { + switch (label.name.toLocaleLowerCase()) { + case 'favorite': + return + case 'pinned': + return + case 'recommended': + return + case 'newsletter': + return + case 'rss': + case 'feed': + return + } + return undefined +} + export const siteName = ( originalArticleUrl: string, itemUrl: string @@ -114,7 +146,10 @@ export function LibraryItemMetadata( }, [props.item.highlights]) return ( - + + {props.item.labels?.map((label) => { + return flairIconForLabel(label) + })} {timeAgo(props.item.savedAt)} {` `} {props.item.wordsCount ?? 0 > 0 @@ -126,7 +161,7 @@ export function LibraryItemMetadata( {highlightCount > 0 ? ` • ${highlightCount} highlight${highlightCount > 1 ? 's' : ''}` : null} - + ) } diff --git a/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx b/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx index 0ba2db164..0d139fbd3 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx @@ -13,6 +13,7 @@ import { siteName, TitleStyle, MenuStyle, + FLAIR_ICON_NAMES, } from './LibraryCardStyles' import { sortedLabels } from '../../../lib/labelsSort' import { LibraryHoverActions } from './LibraryHoverActions' @@ -285,9 +286,14 @@ const LibraryGridCardContent = (props: LinkedItemCardProps): JSX.Element => { marginLeft: '-4px', // offset because the chips have margin }} > - {sortedLabels(props.item.labels).map(({ name, color }, index) => ( - - ))} + {sortedLabels(props.item.labels) + .filter( + ({ name }) => + FLAIR_ICON_NAMES.indexOf(name.toLocaleLowerCase()) == -1 + ) + .map(({ name, color }, index) => ( + + ))} diff --git a/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx b/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx index 95b8c4002..666a0b04c 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx @@ -11,6 +11,7 @@ import { siteName, TitleStyle, MenuStyle, + FLAIR_ICON_NAMES, } from './LibraryCardStyles' import { sortedLabels } from '../../../lib/labelsSort' import { LIBRARY_LEFT_MENU_WIDTH } from '../../templates/homeFeed/LibraryFilterMenu' @@ -354,9 +355,14 @@ export function LibraryListCardContent( display: 'block', }} > - {sortedLabels(props.item.labels).map(({ name, color }, index) => ( - - ))} + {sortedLabels(props.item.labels) + .filter( + ({ name }) => + FLAIR_ICON_NAMES.indexOf(name.toLocaleLowerCase()) == -1 + ) + .map(({ name, color }, index) => ( + + ))} From a95a5ac2a5c8452d6e806e94956ab2b203249a84 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 17:08:24 +0800 Subject: [PATCH 110/115] Add flair icon --- .../elements/icons/FavoriteFlairIcon.tsx | 29 ++++++++++++ .../elements/icons/FeedFlairIcon.tsx | 44 +++++++++++++++++++ .../elements/icons/NewsletterFlairIcon.tsx | 33 ++++++++++++++ .../elements/icons/PinnedFlairIcon.tsx | 29 ++++++++++++ .../elements/icons/RecommendedFlairIcon.tsx | 33 ++++++++++++++ 5 files changed, 168 insertions(+) create mode 100644 packages/web/components/elements/icons/FavoriteFlairIcon.tsx create mode 100644 packages/web/components/elements/icons/FeedFlairIcon.tsx create mode 100644 packages/web/components/elements/icons/NewsletterFlairIcon.tsx create mode 100644 packages/web/components/elements/icons/PinnedFlairIcon.tsx create mode 100644 packages/web/components/elements/icons/RecommendedFlairIcon.tsx diff --git a/packages/web/components/elements/icons/FavoriteFlairIcon.tsx b/packages/web/components/elements/icons/FavoriteFlairIcon.tsx new file mode 100644 index 000000000..ab7ab8386 --- /dev/null +++ b/packages/web/components/elements/icons/FavoriteFlairIcon.tsx @@ -0,0 +1,29 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class FavoriteFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + ) + } +} diff --git a/packages/web/components/elements/icons/FeedFlairIcon.tsx b/packages/web/components/elements/icons/FeedFlairIcon.tsx new file mode 100644 index 000000000..d9001ebbd --- /dev/null +++ b/packages/web/components/elements/icons/FeedFlairIcon.tsx @@ -0,0 +1,44 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class FeedFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + + + ) + } +} diff --git a/packages/web/components/elements/icons/NewsletterFlairIcon.tsx b/packages/web/components/elements/icons/NewsletterFlairIcon.tsx new file mode 100644 index 000000000..47cef57f8 --- /dev/null +++ b/packages/web/components/elements/icons/NewsletterFlairIcon.tsx @@ -0,0 +1,33 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class NewsletterFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + + ) + } +} diff --git a/packages/web/components/elements/icons/PinnedFlairIcon.tsx b/packages/web/components/elements/icons/PinnedFlairIcon.tsx new file mode 100644 index 000000000..f563a93bb --- /dev/null +++ b/packages/web/components/elements/icons/PinnedFlairIcon.tsx @@ -0,0 +1,29 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class PinnedFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + ) + } +} diff --git a/packages/web/components/elements/icons/RecommendedFlairIcon.tsx b/packages/web/components/elements/icons/RecommendedFlairIcon.tsx new file mode 100644 index 000000000..aec665505 --- /dev/null +++ b/packages/web/components/elements/icons/RecommendedFlairIcon.tsx @@ -0,0 +1,33 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class RecommendedFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + + ) + } +} From 973c1dec1ecb7dfd5bb010792d7fedbe7d5731a4 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 17:15:03 +0800 Subject: [PATCH 111/115] When saving PDF read position always set it to the current page --- .../web/components/templates/article/PdfArticleContainer.tsx | 1 + .../lib/networking/mutations/articleReadingProgressMutation.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/web/components/templates/article/PdfArticleContainer.tsx b/packages/web/components/templates/article/PdfArticleContainer.tsx index 210b1b2ae..7e7c30681 100644 --- a/packages/web/components/templates/article/PdfArticleContainer.tsx +++ b/packages/web/components/templates/article/PdfArticleContainer.tsx @@ -413,6 +413,7 @@ export default function PdfArticleContainer( } await articleReadingProgressMutation({ id: props.article.id, + force: true, readingProgressPercent: percent, readingProgressAnchorIndex: pageIndex, }) diff --git a/packages/web/lib/networking/mutations/articleReadingProgressMutation.ts b/packages/web/lib/networking/mutations/articleReadingProgressMutation.ts index 6c41950d0..95fd754d7 100644 --- a/packages/web/lib/networking/mutations/articleReadingProgressMutation.ts +++ b/packages/web/lib/networking/mutations/articleReadingProgressMutation.ts @@ -3,6 +3,7 @@ import { gqlFetcher } from '../networkHelpers' export type ArticleReadingProgressMutationInput = { id: string + force?: boolean readingProgressPercent?: number readingProgressTopPercent?: number readingProgressAnchorIndex?: number From 95c984edf132ebf474142b8777e1db1b90cd1d8b Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 17:18:26 +0800 Subject: [PATCH 112/115] Remove soptimization --- .../web/components/templates/article/PdfArticleContainer.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/web/components/templates/article/PdfArticleContainer.tsx b/packages/web/components/templates/article/PdfArticleContainer.tsx index 7e7c30681..4c8ba5bbd 100644 --- a/packages/web/components/templates/article/PdfArticleContainer.tsx +++ b/packages/web/components/templates/article/PdfArticleContainer.tsx @@ -408,9 +408,6 @@ export default function PdfArticleContainer( 100, Math.max(0, ((pageIndex + 1) / instance.totalPageCount) * 100) ) - if (percent <= props.article.readingProgressPercent) { - return - } await articleReadingProgressMutation({ id: props.article.id, force: true, From 769f3b6b03fbf1c0814b9fd2b4fe9a50e905f127 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 17:25:44 +0800 Subject: [PATCH 113/115] Fix svgs --- .../web/components/elements/icons/FeedFlairIcon.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/web/components/elements/icons/FeedFlairIcon.tsx b/packages/web/components/elements/icons/FeedFlairIcon.tsx index d9001ebbd..1b1c4b78f 100644 --- a/packages/web/components/elements/icons/FeedFlairIcon.tsx +++ b/packages/web/components/elements/icons/FeedFlairIcon.tsx @@ -22,20 +22,20 @@ export class FeedFlairIcon extends React.Component { d="M8.73861 3.23242L3.40527 5.89909L8.73861 8.56576L14.0719 5.89909L8.73861 3.23242Z" fill="#FF7B03" stroke="#FF7B03" - stroke-linecap="round" - stroke-linejoin="round" + strokeLinecap="round" + strokeLinejoin="round" /> From b57fd742b34a5b6216dee3b1ae2c8cf3ca325529 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 18:05:06 +0800 Subject: [PATCH 114/115] Add titles to the flair icons --- .../elements/icons/FavoriteFlairIcon.tsx | 1 + .../LibraryCards/LibraryCardStyles.tsx | 30 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/web/components/elements/icons/FavoriteFlairIcon.tsx b/packages/web/components/elements/icons/FavoriteFlairIcon.tsx index ab7ab8386..9f2ec3b87 100644 --- a/packages/web/components/elements/icons/FavoriteFlairIcon.tsx +++ b/packages/web/components/elements/icons/FavoriteFlairIcon.tsx @@ -1,5 +1,6 @@ /* eslint-disable functional/no-class */ /* eslint-disable functional/no-this-expression */ +import { SpanBox } from '../LayoutPrimitives' import { IconProps } from './IconProps' import React from 'react' diff --git a/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx b/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx index e7fd81c5a..a27570122 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx @@ -101,16 +101,36 @@ export const FLAIR_ICON_NAMES = [ const flairIconForLabel = (label: Label): JSX.Element | undefined => { switch (label.name.toLocaleLowerCase()) { case 'favorite': - return + return ( + + + + ) case 'pinned': - return + return ( + + + + ) case 'recommended': - return + return ( + + + + ) case 'newsletter': - return + return ( + + + + ) case 'rss': case 'feed': - return + return ( + + + + ) } return undefined } From 65b20fcb866f41c4c360b168a7d3d684595fe093 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Tue, 24 Oct 2023 18:13:32 +0800 Subject: [PATCH 115/115] Revert unneeded import --- packages/web/components/elements/icons/FavoriteFlairIcon.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/web/components/elements/icons/FavoriteFlairIcon.tsx b/packages/web/components/elements/icons/FavoriteFlairIcon.tsx index 9f2ec3b87..ab7ab8386 100644 --- a/packages/web/components/elements/icons/FavoriteFlairIcon.tsx +++ b/packages/web/components/elements/icons/FavoriteFlairIcon.tsx @@ -1,6 +1,5 @@ /* eslint-disable functional/no-class */ /* eslint-disable functional/no-this-expression */ -import { SpanBox } from '../LayoutPrimitives' import { IconProps } from './IconProps' import React from 'react'