diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5324e46ef..3ef5816c6 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,40 +1,29 @@ -Guidelines -========== +📝 **Guidelines for Contribution** -When contributing to Omnivore, please follow the style of the file you -are editing. For most code running `yarn lint` should tell you if -your code meets are style. +Hello there! When contributing to Omnivore, it's essential to maintain a style consistent with the file you're editing. You can quickly check if your code aligns with our style by running `yarn lint`. Your attention to detail here is appreciated! 😊 +📜 **License** -License -======= +Omnivore is proudly licensed under the [AGPL license](https://github.com/omnivore-app/omnivore/blob/main/LICENSE). Let's keep our code open and accessible to all! -Omnivore is licensed under the [AGPL license](https://github.com/omnivore-app/omnivore/blob/main/LICENSE). +🤝 **CLA (Contributor License Agreement)** -CLA -======= +We kindly request all contributors to adhere to Omnivore's CLA, which you can review and accept at [Omnivore's CLA](https://cla-assistant.io/omnivore-app/omnivore). Your commitment to this agreement is much appreciated. -Contributions are accepted under [Omnivore's CLA](https://cla-assistant.io/omnivore-app/omnivore). +👩‍💻 **Code Review** -Code Review -======= -We require code review from a [CODEOWNER](https://github.com/omnivore-app/omnivore/blob/main/.github/CODEOWNERS) before merging a pull request. +Before merging any pull request, we require a code review from one of our trusted [CODEOWNERS](https://github.com/omnivore-app/omnivore/blob/main/.github/CODEOWNERS). It ensures the quality and integrity of our codebase. -Testing -======= +🧪 **Testing** -Pull requests are automatically tested using GitHub Actions. We -usually only merge a pull request if it causes no regressions in -our tests. +We take quality seriously! Pull requests automatically undergo testing using GitHub Actions. We typically merge a pull request only if it passes all our tests without causing any regressions. 🧪 -When you submit a pull request: +📬 **Submitting a Pull Request** -* If you are a new contributor, GitHub will ask for permissions (on - the pull request) to test it. A maintainer will reply to approve - the test run if they find the patch appropriate. -* If you have previously contributed, GitHub will test your pull - request as soon as a test machine is available. +- **New Contributors**: If you're a new contributor, GitHub may request permissions on the pull request to run tests. Don't worry, our maintainer will swiftly approve the test run if your patch looks promising. -Once tests are passing on your pull request it will be reviewed, -merged, and deployed by a maintiner. We deploy around ten times -a day, so your changes should hit production quickly. +- **Previous Contributors**: For those who've contributed before, GitHub will kick off testing as soon as a test machine is available. + +Once your pull request passes the tests, it'll be promptly reviewed, merged, and deployed by one of our diligent maintainers. We usually deploy changes around ten times a day, so your contributions will go live quickly! 🚀 + +Thank you for being part of the Omnivore community. Let's keep coding, collaborating, and creating something amazing together! 🌟 \ No newline at end of file diff --git a/README.md b/README.md index 54bf3b2b4..111cdb065 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ We built Omnivore because we love reading and we want it to be more social. Join - Add newsletter articles via email (with substack support!) - PDF support - [Web app](https://omnivore.app/) written in Node.js and TypeScript -- [Native iOS app](https://omnivore.app/install/ios) +- [Native iOS app](https://omnivore.app/install/ios) ([source](https://github.com/omnivore-app/omnivore/tree/main/apple)) - [Android app](https://omnivore.app/install/android) ([source](https://github.com/omnivore-app/omnivore/tree/main/android/Omnivore)) - Progressive web app for Android users - Browser extensions for [Chrome](https://omnivore.app/install/chrome), [Safari](https://omnivore.app/install/safari), [Firefox](https://omnivore.app/install/firefox), and [Edge](https://omnivore.app/install/edge) @@ -88,7 +88,7 @@ Open and confirm Omnivore is running ### 3. Login with the test account -During database setup docker compose creates an account `demo@omnivore.app`, password: `demo`. +During database setup docker compose creates an account `demo@omnivore.app`, password: `demo_password`. Go to in your browser and choose `Continue with Email` to login. diff --git a/android/Omnivore/app/build.gradle b/android/Omnivore/app/build.gradle index e525d3c63..beeda4ff1 100644 --- a/android/Omnivore/app/build.gradle +++ b/android/Omnivore/app/build.gradle @@ -17,8 +17,8 @@ android { applicationId "app.omnivore.omnivore" minSdk 26 targetSdk 33 - versionCode 110 - versionName "0.0.110" + versionCode 122 + versionName "0.0.122" testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" vectorDrawables { diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemLabelMutations.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemLabelMutations.kt index e7120536f..e4dd3561f 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemLabelMutations.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemLabelMutations.kt @@ -5,12 +5,12 @@ import app.omnivore.omnivore.graphql.generated.SetLabelsMutation import app.omnivore.omnivore.graphql.generated.type.CreateLabelInput import app.omnivore.omnivore.graphql.generated.type.SetLabelsInput -suspend fun Networker.updateLabelsForSavedItem(input: SetLabelsInput): Boolean { +suspend fun Networker.updateLabelsForSavedItem(input: SetLabelsInput): List? { return try { val result = authenticatedApolloClient().mutation(SetLabelsMutation(input)).execute() - return result.data?.setLabels?.onSetLabelsSuccess?.labels != null + return result.data?.setLabels?.onSetLabelsSuccess?.labels } catch (e: java.lang.Exception) { - false + return null } } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemQuery.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemQuery.kt index 2fd8e9298..cd2f50e35 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemQuery.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/networking/SavedItemQuery.kt @@ -1,9 +1,15 @@ package app.omnivore.omnivore.networking +import android.util.Log import app.omnivore.omnivore.graphql.generated.GetArticleQuery +import app.omnivore.omnivore.graphql.generated.type.ContentReader import app.omnivore.omnivore.persistence.entities.SavedItem import app.omnivore.omnivore.persistence.entities.SavedItemLabel import app.omnivore.omnivore.persistence.entities.Highlight +import java.io.File +import java.net.URL +import java.nio.file.Files +import java.nio.file.StandardCopyOption data class SavedItemQueryResponse( val item: SavedItem?, @@ -60,7 +66,18 @@ suspend fun Networker.savedItem(slug: String): SavedItemQueryResponse { ) } - // TODO: handle errors + var localPDFPath: String? = null + if (article.articleFields.contentReader == ContentReader.PDF) { + // download the PDF and save it locally + // article.articleFields.url + + val localFile = File.createTempFile("pdf-" + article.articleFields.id, ".pdf", ) + val url = URL(article.articleFields.url) + Log.d("pdf", "creating local file: $localFile") + + url.openStream().use { Files.copy(it, localFile.toPath(), StandardCopyOption.REPLACE_EXISTING) } + localPDFPath = localFile.toPath().toString() + } val savedItem = SavedItem( savedItemId = article.articleFields.id, @@ -82,7 +99,8 @@ suspend fun Networker.savedItem(slug: String): SavedItemQueryResponse { isArchived = article.articleFields.isArchived, contentReader = article.articleFields.contentReader.rawValue, content = article.articleFields.content, - wordsCount = article.articleFields.wordsCount + wordsCount = article.articleFields.wordsCount, + localPDFPath = localPDFPath ) return SavedItemQueryResponse(item = savedItem, highlights, labels = savedItemLabels, state = article.articleFields.state?.rawValue ?: "") diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/AppDatabase.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/AppDatabase.kt index 07202993f..bf98747fd 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/AppDatabase.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/AppDatabase.kt @@ -13,7 +13,7 @@ import app.omnivore.omnivore.persistence.entities.* SavedItemAndSavedItemLabelCrossRef::class, SavedItemAndHighlightCrossRef::class ], - version = 11 + version = 12 ) abstract class AppDatabase : RoomDatabase() { abstract fun viewerDao(): ViewerDao diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt index 863ebe7ae..7dbd1dfd0 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt @@ -40,7 +40,8 @@ data class SavedItem( @ColumnInfo(typeAffinity = ColumnInfo.BLOB) val pdfData: ByteArray? = null, var serverSyncStatus: Int = 0, val tempPDFURL: String? = null, - val wordsCount: Int? = null + val wordsCount: Int? = null, + val localPDFPath: String? = null // hasMany highlights // hasMany labels diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt index 0461149a7..7dc98a9f7 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsSelectionSheet.kt @@ -3,6 +3,7 @@ package app.omnivore.omnivore.ui.components import LabelChip +import android.widget.Toast import androidx.compose.foundation.* import androidx.compose.foundation.interaction.FocusInteraction import androidx.compose.foundation.interaction.MutableInteractionSource @@ -219,12 +220,12 @@ class LabelChipView(label: SavedItemLabel) : Chip(label.name) { val label = label } -fun findOrCreateLabel(labelsViewModel: LabelsViewModel, labels: List, name: TextFieldValue): SavedItemLabel { - val found = labels.find { it.name == name.text } +fun findOrCreateLabel(labelsViewModel: LabelsViewModel, labels: List, name: String): SavedItemLabel { + val found = labels.find { it.name == name } if (found != null) { return found } - return labelsViewModel.createNewSavedItemLabelWithTemp(name.text, LabelSwatchHelper.random()) + return labelsViewModel.createNewSavedItemLabelWithTemp(name, LabelSwatchHelper.random()) } @Composable @@ -308,7 +309,7 @@ fun LabelsSelectionSheetContent( LabelChipView(it) } ?: null } else { - LabelChipView(findOrCreateLabel(labelsViewModel = labelsViewModel, labels = labels, name = it)) + LabelChipView(findOrCreateLabel(labelsViewModel = labelsViewModel, labels = labels, name = it.text)) } }, chipLeadingIcon = { chip -> CircleIcon(colorHex = chip.label.color) }, @@ -341,19 +342,34 @@ fun LabelsSelectionSheetContent( ) if (!isLibraryMode && filterTextValue.text.isNotEmpty() && currentLabel == null) { + val context = LocalContext.current Row( horizontalArrangement = Arrangement.Start, verticalAlignment = Alignment.CenterVertically, modifier = Modifier .fillMaxWidth() .clickable { - val label = findOrCreateLabel( - labelsViewModel = labelsViewModel, - labels = labels, - name = filterTextValue - ) - state.addChip(LabelChipView(label)) - filterTextValue = TextFieldValue() + val labelName = filterTextValue.text.trim() + when(labelsViewModel.validateLabelName(labelName)) { + LabelsViewModel.Error.LabelNameTooLong -> { + Toast.makeText( + context, + context.getString(R.string.label_selection_sheet_label_too_long_error_msg, + labelsViewModel.labelNameMaxLength), + Toast.LENGTH_SHORT + ).show() + } + null -> { + val label = findOrCreateLabel( + labelsViewModel = labelsViewModel, + labels = labels, + name = labelName + ) + + state.addChip(LabelChipView(label)) + filterTextValue = TextFieldValue() + } + } } .padding(horizontal = 10.dp) .padding(top = 10.dp, bottom = 5.dp) @@ -364,7 +380,7 @@ fun LabelsSelectionSheetContent( contentDescription = null, modifier = Modifier.padding(end = 8.dp) ) - Text(text = stringResource(R.string.label_selection_sheet_text_create, filterTextValue.text)) + Text(text = stringResource(R.string.label_selection_sheet_text_create, filterTextValue.text.trim())) } } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt index f75959f70..c9dab6773 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/components/LabelsViewModel.kt @@ -1,34 +1,16 @@ package app.omnivore.omnivore.ui.components - -import android.content.ClipData -import android.content.ClipboardManager -import android.content.Context -import android.content.Intent -import android.net.Uri import android.util.Log -import android.widget.Toast -import androidx.compose.ui.platform.LocalContext -import androidx.core.content.ContextCompat.startActivity import androidx.lifecycle.* -import app.omnivore.omnivore.DatastoreKeys import app.omnivore.omnivore.DatastoreRepository import app.omnivore.omnivore.dataService.* import app.omnivore.omnivore.graphql.generated.type.CreateLabelInput -import app.omnivore.omnivore.graphql.generated.type.SetLabelsInput import app.omnivore.omnivore.models.ServerSyncStatus import app.omnivore.omnivore.networking.* -import app.omnivore.omnivore.persistence.entities.SavedItem -import app.omnivore.omnivore.persistence.entities.SavedItemAndSavedItemLabelCrossRef import app.omnivore.omnivore.persistence.entities.SavedItemLabel -import app.omnivore.omnivore.ui.components.LabelSwatchHelper -import app.omnivore.omnivore.ui.library.SavedItemAction import com.apollographql.apollo3.api.Optional.Companion.presentIfNotNull -import com.google.gson.Gson import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.* -import kotlinx.coroutines.flow.distinctUntilChanged -import java.time.Instant import java.time.LocalDate import java.time.ZoneOffset import java.time.format.DateTimeFormatter @@ -42,6 +24,24 @@ class LabelsViewModel @Inject constructor( private val dataService: DataService, private val networker: Networker ): ViewModel() { + val labelNameMaxLength = 64 + + enum class Error { + LabelNameTooLong + } + + /** + * Checks whether or not the provided label name is valid. + * @param labelName The name of the label. + * @return null if valid, [Error] otherwise. + */ + fun validateLabelName(labelName: String): Error? { + if (labelName.count() > labelNameMaxLength) { + return Error.LabelNameTooLong + } + + return null + } fun createNewSavedItemLabelWithTemp(labelName: String, hexColorValue: String): SavedItemLabel { val tempId = UUID.randomUUID().toString() @@ -56,21 +56,6 @@ class LabelsViewModel @Inject constructor( serverSyncStatus = ServerSyncStatus.NEEDS_CREATION.rawValue ) - viewModelScope.launch { - withContext(Dispatchers.IO) { - dataService.db.savedItemLabelDao().insertAll(listOf(res)) - - val newLabel = networker.createNewLabel(CreateLabelInput(color = presentIfNotNull(res.color), name = res.name)) - if (newLabel != null) { - try { - dataService.db.savedItemLabelDao().updateTempLabel(tempId, newLabel.id) - } catch (e: Exception) { - Log.d("EXCEPTION: ", e.toString()) - } - } - } - } - return res } } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt index 632b32950..86dfab019 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt @@ -304,55 +304,44 @@ class LibraryViewModel @Inject constructor( fun updateSavedItemLabels(savedItemID: String, labels: List) { viewModelScope.launch { withContext(Dispatchers.IO) { - val syncedLabels = labels.filter { it.serverSyncStatus == ServerSyncStatus.IS_SYNCED.rawValue } - val unsyncedLabels = labels.filter { it.serverSyncStatus != ServerSyncStatus.IS_SYNCED.rawValue } + val input = SetLabelsInput( + pageId = savedItemID, + labels = Optional.presentIfNotNull(labels.map { CreateLabelInput(color = Optional.presentIfNotNull(it.color), name = it.name) }), + ) - var labelCreationError = false - val createdLabels = unsyncedLabels.mapNotNull { label -> - val result = networker.createNewLabel(CreateLabelInput( - name = label.name, - color = presentIfNotNull(label.color), - description = presentIfNotNull(label.labelDescription), - )) - result?.let { + val updatedLabels = networker.updateLabelsForSavedItem(input) + + // Figure out which of the labels are new + updatedLabels?.let { updatedLabels -> + val existingNamedLabels = dataService.db.savedItemLabelDao() + .namedLabels(updatedLabels.map { it.labelFields.name }) + val existingNames = existingNamedLabels.map { it.name } + val newNamedLabels = updatedLabels.filter { !existingNames.contains(it.labelFields.name) } + + dataService.db.savedItemLabelDao().insertAll(newNamedLabels.map { SavedItemLabel( - savedItemLabelId = result.id, - name = result.name, - color = result.color, - createdAt = result.createdAt.toString(), - labelDescription = result.description, - serverSyncStatus = ServerSyncStatus.IS_SYNCED.rawValue + savedItemLabelId = it.labelFields.id, + name = it.labelFields.name, + color = it.labelFields.color, + createdAt = null, + labelDescription = null + ) + }) + + val allNamedLabels = dataService.db.savedItemLabelDao() + .namedLabels(updatedLabels.map { it.labelFields.name }) + val crossRefs = allNamedLabels.map { + SavedItemAndSavedItemLabelCrossRef( + savedItemLabelId = it.savedItemLabelId, + savedItemId = savedItemID ) - } ?: run { - labelCreationError = true - null } - } + dataService.db.savedItemAndSavedItemLabelCrossRefDao().deleteRefsBySavedItemId(savedItemID) + dataService.db.savedItemAndSavedItemLabelCrossRefDao().insertAll(crossRefs) - dataService.db.savedItemLabelDao().insertAll(createdLabels) - - val allLabels = syncedLabels + createdLabels - - val input = SetLabelsInput(labelIds = Optional.presentIfNotNull(allLabels.map { it.savedItemLabelId }), pageId = savedItemID) - val networkResult = networker.updateLabelsForSavedItem(input) - - val crossRefs = allLabels.map { - SavedItemAndSavedItemLabelCrossRef( - savedItemLabelId = it.savedItemLabelId, - savedItemId = savedItemID - ) - } - - // Remove all labels first - dataService.db.savedItemAndSavedItemLabelCrossRefDao().deleteRefsBySavedItemId(savedItemID) - - // Add back the current labels - dataService.db.savedItemAndSavedItemLabelCrossRefDao().insertAll(crossRefs) - - if (!networkResult || labelCreationError) { - snackbarMessage = resourceProvider.getString(R.string.library_view_model_snackbar_error) - } else { snackbarMessage = resourceProvider.getString(R.string.library_view_model_snackbar_success) + } ?: run { + snackbarMessage = resourceProvider.getString(R.string.library_view_model_snackbar_error) } CoroutineScope(Dispatchers.Main).launch { diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt new file mode 100644 index 000000000..4ac6f8545 --- /dev/null +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/OpenLinkView.kt @@ -0,0 +1,59 @@ +package app.omnivore.omnivore.ui.reader + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.Button +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import app.omnivore.omnivore.R + +@Composable +fun OpenLinkView(webReaderViewModel: WebReaderViewModel) { + val context = LocalContext.current + + Column(modifier = Modifier + .padding(top = 25.dp) + .padding(horizontal = 50.dp), verticalArrangement = Arrangement.spacedBy(20.dp)) { + Row { + Text(webReaderViewModel.currentLink.toString(), + fontWeight = FontWeight.Light, + color = Color.DarkGray, + fontSize = 12.sp, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } + Row(modifier = Modifier.padding(top = 25.dp)) { + Button(onClick = { webReaderViewModel.openCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_open_in_browser)) + } + } + Row { + Button(onClick = { webReaderViewModel.saveCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_save_to_omnivore)) + + } + } + Row { + Button(onClick = {webReaderViewModel.copyCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_copy_link)) + } + } + Row { + Button(onClick = {webReaderViewModel.resetBottomSheet() }, modifier = Modifier.fillMaxWidth()) { + Text(text = stringResource(R.string.open_link_view_action_cancel)) + } + } + } +} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/PDFReaderViewModel.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/PDFReaderViewModel.kt index 056fa559c..e674b89ac 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/PDFReaderViewModel.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/PDFReaderViewModel.kt @@ -7,7 +7,7 @@ import androidx.lifecycle.MutableLiveData import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import app.omnivore.omnivore.DatastoreRepository -import app.omnivore.omnivore.EventTracker +import app.omnivore.omnivore.dataService.DataService import app.omnivore.omnivore.graphql.generated.type.CreateHighlightInput import app.omnivore.omnivore.graphql.generated.type.MergeHighlightInput import app.omnivore.omnivore.graphql.generated.type.UpdateHighlightInput @@ -21,11 +21,16 @@ import com.pspdfkit.document.download.DownloadJob import com.pspdfkit.document.download.DownloadRequest import com.pspdfkit.document.download.Progress import dagger.hilt.android.lifecycle.HiltViewModel +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import org.json.JSONObject import java.io.File import java.lang.Double.max import java.lang.Double.min +import java.lang.Exception +import java.net.URLEncoder +import java.nio.file.FileSystem import java.util.* import javax.inject.Inject @@ -38,8 +43,8 @@ data class PDFReaderParams( @HiltViewModel class PDFReaderViewModel @Inject constructor( private val datastoreRepo: DatastoreRepository, - private val networker: Networker, - private val eventTracker: EventTracker, + private val dataService: DataService, + private val networker: Networker ): ViewModel() { var annotationUnderNoteEdit: Annotation? = null val pdfReaderParamsLiveData = MutableLiveData(null) @@ -47,21 +52,51 @@ class PDFReaderViewModel @Inject constructor( fun loadItem(slug: String, context: Context) { viewModelScope.launch { + loadItemFromDB(slug) + loadItemFromNetwork(slug, context) + } + } + + private suspend fun loadItemFromDB(slug: String) { + withContext(Dispatchers.IO) { + val persistedItem = dataService.db.savedItemDao().getSavedItemWithLabelsAndHighlights(slug) + persistedItem?.let { persistedItem -> + persistedItem?.savedItem?.localPDF?.let { localPDF -> + val localFile = File(localPDF) + + if (localFile.exists()) { + val articleContent = ArticleContent( + title = persistedItem.savedItem.title, + htmlContent = "", + highlights = persistedItem.highlights, + contentStatus = "SUCCEEDED", + objectID = "", + labelsJSONString = Gson().toJson(persistedItem.labels) + ) + + pdfReaderParamsLiveData.postValue( + PDFReaderParams( + persistedItem.savedItem, + articleContent, + Uri.fromFile(localFile) + ) + ) + } + } + } + } + } + + private suspend fun loadItemFromNetwork(slug: String, context: Context) { + withContext(Dispatchers.IO) { val articleQueryResult = networker.savedItem(slug) - - val article = articleQueryResult.item ?: return@launch - + val article = articleQueryResult.item ?: return@withContext val request = DownloadRequest.Builder(context) .uri(article.pageURLString) .build() val job = DownloadJob.startDownload(request) - job.setProgressListener(object : DownloadJob.ProgressListenerAdapter() { - override fun onProgress(progress: Progress) { -// progressBar.setProgress((100f * progress.bytesReceived / progress.totalBytes).toInt()) - } - override fun onComplete(output: File) { val articleContent = ArticleContent( title = article.title, @@ -72,22 +107,18 @@ class PDFReaderViewModel @Inject constructor( labelsJSONString = Gson().toJson(articleQueryResult.labels) ) - val pdfReaderParams = PDFReaderParams(article, articleContent, Uri.fromFile(output)) - - eventTracker.track("link_read", - com.posthog.android.Properties() - .putValue("linkID", pdfReaderParams.item.savedItemId) - .putValue("slug", pdfReaderParams.item.slug) - .putValue("originalArticleURL", pdfReaderParams.item.pageURLString) - .putValue("loaded_from", "network") - ) - currentReadingProgress = article.readingProgress - pdfReaderParamsLiveData.postValue(pdfReaderParams) + pdfReaderParamsLiveData.postValue( + PDFReaderParams( + article, + articleContent, + Uri.fromFile(output) + ) + ) } override fun onError(exception: Throwable) { -// handleDownloadError(exception) +// handleDownloadError(exception) } }) } diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderContent.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderContent.kt index 571e423d7..e04f13591 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderContent.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderContent.kt @@ -91,7 +91,7 @@ data class WebReaderContent( url: `${item.pageURLString}`, title: `${articleContent.title.replace("`", "\\`")}`, content: document.getElementById('_omnivore-htmlContent').innerHTML, - originalArticleUrl: "${item.pageURLString}", + originalArticleUrl: "${item.publisherURLString}", contentReader: "WEB", readingProgressPercent: ${item.readingProgress}, readingProgressAnchorIndex: ${item.readingProgressAnchor}, diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt index e680a42a7..87a0d63ed 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt @@ -42,7 +42,6 @@ import com.google.accompanist.systemuicontroller.rememberSystemUiController import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.launch import kotlin.math.roundToInt -import androidx.compose.material3.Button import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import app.omnivore.omnivore.ui.components.LabelsViewModel @@ -450,38 +449,3 @@ fun BottomSheetUI(title: String?, content: @Composable () -> Unit) { } } } - -@OptIn(ExperimentalMaterial3Api::class, ExperimentalMaterialApi::class) -@Composable -fun OpenLinkView(webReaderViewModel: WebReaderViewModel) { - val context = LocalContext.current - - Column(modifier = Modifier - .padding(top = 50.dp) - .padding(horizontal = 50.dp), verticalArrangement = Arrangement.spacedBy(20.dp)) { - Row { - Button(onClick = { webReaderViewModel.openCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_open_in_browser)) - - } - } - Row() { - Button(onClick = { webReaderViewModel.saveCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_save_to_omnivore)) - - } - } - Row() { - Button(onClick = {webReaderViewModel.copyCurrentLink(context) }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_copy_link)) - - } - } - Row { - Button(onClick = {webReaderViewModel.resetBottomSheet() }, modifier = Modifier.fillMaxWidth()) { - Text(text = stringResource(R.string.open_link_view_action_cancel)) - - } - } - } -} diff --git a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/savedItemViews/SavedItemCard.kt b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/savedItemViews/SavedItemCard.kt index 654fcd22c..be0de7ba5 100644 --- a/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/savedItemViews/SavedItemCard.kt +++ b/android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/savedItemViews/SavedItemCard.kt @@ -20,16 +20,20 @@ import androidx.compose.ui.focus.onFocusChanged import androidx.compose.ui.graphics.Color import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.colorResource +import androidx.compose.ui.res.painterResource import androidx.compose.ui.text.TextStyle import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.intl.Locale import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.text.toLowerCase +import androidx.compose.ui.text.toUpperCase import androidx.compose.ui.unit.* import app.omnivore.omnivore.R +import app.omnivore.omnivore.persistence.entities.SavedItemLabel import app.omnivore.omnivore.persistence.entities.SavedItemWithLabelsAndHighlights import app.omnivore.omnivore.ui.components.LabelChipColors import app.omnivore.omnivore.ui.library.SavedItemAction +import app.omnivore.omnivore.ui.library.SavedItemFilter import app.omnivore.omnivore.ui.library.SavedItemViewModel import coil.compose.rememberAsyncImagePainter @@ -45,14 +49,14 @@ fun SavedItemCard( Column( modifier = Modifier - .combinedClickable( - onClick = onClickHandler, - onLongClick = { - savedItemViewModel.actionsMenuItemLiveData.postValue(savedItem) - } - ) - .background(if (selected) MaterialTheme.colorScheme.surfaceVariant else MaterialTheme.colorScheme.background) - .fillMaxWidth() + .combinedClickable( + onClick = onClickHandler, + onLongClick = { + savedItemViewModel.actionsMenuItemLiveData.postValue(savedItem) + } + ) + .background(if (selected) MaterialTheme.colorScheme.surfaceVariant else MaterialTheme.colorScheme.background) + .fillMaxWidth() ) { Row( horizontalArrangement = Arrangement.SpaceBetween, @@ -108,8 +112,10 @@ fun SavedItemCard( ) } - FlowRow(modifier = Modifier.fillMaxWidth().padding(10.dp)) { - savedItem.labels.sortedWith(compareBy { it.name.toLowerCase(Locale.current) }).forEach { label -> + FlowRow(modifier = Modifier + .fillMaxWidth() + .padding(10.dp)) { + savedItem.labels.filter { !isFlairLabel(it) }.sortedWith(compareBy { it.name.toLowerCase(Locale.current) }).forEach { label -> val chipColors = LabelChipColors.fromHex(label.color) LabelChip( @@ -196,6 +202,82 @@ fun readingProgress(item: SavedItemWithLabelsAndHighlights): String { // return "" //} + +public enum class FlairIcon( + public val rawValue: String, + public val sortOrder: Int +) { + FEED("feed", 0), + RSS("rss", 0), + FAVORITE("favorite", 1), + NEWSLETTER("newsletter", 2), + RECOMMENDED("recommended", 3), + PINNED("pinned", 4) +} + +val FLAIR_ICON_NAMES = listOf("feed", "rss", "favorite", "newsletter", "recommended", "pinned") + +fun isFlairLabel(label: SavedItemLabel): Boolean { + return FLAIR_ICON_NAMES.contains(label.name.toLowerCase(Locale.current)) +} + +@Composable +fun flairIcons(item: SavedItemWithLabelsAndHighlights) { + val labels = item.labels.filter { isFlairLabel(it) }.map { + FlairIcon.valueOf(it.name.toUpperCase(Locale.current)) + } + labels.forEach { + when (it) { + FlairIcon.RSS, + FlairIcon.FEED -> { + Image( + painter = painterResource(id = R.drawable.flair_feed), + contentDescription = "Feed flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + FlairIcon.FAVORITE -> { + Image( + painter = painterResource(id = R.drawable.flaire_favorite), + contentDescription = "Favorite flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + FlairIcon.NEWSLETTER -> { + Image( + painter = painterResource(id = R.drawable.flair_newsletter), + contentDescription = "Newsletter flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + FlairIcon.RECOMMENDED -> { + Image( + painter = painterResource(id = R.drawable.flair_recommended), + contentDescription = "Recommended flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + FlairIcon.PINNED -> { + Image( + painter = painterResource(id = R.drawable.flair_pinned), + contentDescription = "Pinned flair Icon", + modifier = Modifier + .padding(end = 5.0.dp) + ) + } + + } + } +} + @Composable fun readInfo(item: SavedItemWithLabelsAndHighlights) { Row( @@ -203,6 +285,9 @@ fun readInfo(item: SavedItemWithLabelsAndHighlights) { .fillMaxWidth() .defaultMinSize(minHeight = 15.dp) ) { + // Show flair here + flairIcons(item) + Text( text = estimatedReadingTime(item), style = TextStyle( diff --git a/android/Omnivore/app/src/main/res/drawable/flair_feed.xml b/android/Omnivore/app/src/main/res/drawable/flair_feed.xml new file mode 100644 index 000000000..aca102b59 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flair_feed.xml @@ -0,0 +1,31 @@ + + + + + + + + diff --git a/android/Omnivore/app/src/main/res/drawable/flair_newsletter.xml b/android/Omnivore/app/src/main/res/drawable/flair_newsletter.xml new file mode 100644 index 000000000..5c70af531 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flair_newsletter.xml @@ -0,0 +1,16 @@ + + + + + + + diff --git a/android/Omnivore/app/src/main/res/drawable/flair_pinned.xml b/android/Omnivore/app/src/main/res/drawable/flair_pinned.xml new file mode 100644 index 000000000..25b42df77 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flair_pinned.xml @@ -0,0 +1,13 @@ + + + + + + diff --git a/android/Omnivore/app/src/main/res/drawable/flair_recommended.xml b/android/Omnivore/app/src/main/res/drawable/flair_recommended.xml new file mode 100644 index 000000000..88a5ea8d3 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flair_recommended.xml @@ -0,0 +1,16 @@ + + + + + + + diff --git a/android/Omnivore/app/src/main/res/drawable/flaire_favorite.xml b/android/Omnivore/app/src/main/res/drawable/flaire_favorite.xml new file mode 100644 index 000000000..820704635 --- /dev/null +++ b/android/Omnivore/app/src/main/res/drawable/flaire_favorite.xml @@ -0,0 +1,13 @@ + + + + + + diff --git a/android/Omnivore/app/src/main/res/values-zh-rTW/strings.xml b/android/Omnivore/app/src/main/res/values-zh-rTW/strings.xml new file mode 100644 index 000000000..ff6db1f70 --- /dev/null +++ b/android/Omnivore/app/src/main/res/values-zh-rTW/strings.xml @@ -0,0 +1,207 @@ + + Omnivore + 絕不錯過精彩閱讀 + 深入了解 + 在無干擾的閱讀器中儲存文章，以便稍後閱讀。 + 標記 + 複製 + 註解 + 移除 + 標記 + 複製 + 註解 + 複製 + 註解 + + + 使用 Apple 繼續 + 正在登入... + + + 建立您的個人檔案 + 載入中... + 取消註冊 + 送出 + 名稱 + 名稱 + 使用者名稱 + 使用者名稱 + 請輸入有效的名稱和使用者名稱。 + + + 載入中... + 返回社交登入頁面 + 還沒有帳戶？ + 忘記密碼？ + 登入 + user@email.com + 電子郵件 + 密碼 + 密碼 + 請輸入電子郵件地址和密碼。 + + + 我們已向 %1$s 傳送驗證電子郵件。請驗證您的電子郵件，然後點選下面的按鈕。 + 檢查狀態 + 使用不同的電子郵件？ + 載入中... + 返回社交登入頁面 + 已經有帳戶？ + 註冊 + user@email.com + 電子郵件 + 密碼 + 密碼 + 名稱 + 名稱 + 使用者名稱 + 使用者名稱 + 請完成所有欄位。 + + + 使用 Google 繼續 + 正在登入... + + + 自建伺服器設定已更新。 + 自建伺服器設定已重設。 + 使用者名稱必須介於 4 到 15 個字元之間。 + 使用者名稱只能包含字母和數字。 + 此使用者名稱不可用。 + 抱歉，我們無法連線到伺服器。 + 出了些問題。請檢查您的電子郵件/密碼，然後再試一次。 + 出了些問題。請檢查您的登入資訊，然後再試一次。 + 無法使用 Google 進行身份驗證。 + 找不到身份驗證權杖。 + + + 載入中... + 重設 + 返回 + 儲存 + 了解更多關於自建伺服器 Omnivore 的資訊 + API 伺服器 + Web 伺服器 + 請輸入 API 伺服器和 Web 伺服器地址。 + + + 忽略 + 使用電子郵件繼續 + 自建伺服器選項 + + + 建立新標籤 + 指定名稱和顏色。 + 建立 + 取消 + 標籤名稱 + + + 按標籤篩選 + 設定標籤 + 取消 + 搜尋 + 儲存 + 建立名為 \"%1$s\" 的新標籤 + 提供的名稱太長（必須小於或等於 %1$d 個字元） + + + 標籤 + + + 圖書館 + + 搜尋 + + + 標籤已更新 + 無法設定標籤 + + + 筆記本 + 複製 + 筆記本已複製 + + + 註解 + 儲存 + 取消 + + + 文章註解 + 新增註解... + + + 標記 + 複製 + 標記已複製 + 新增註解... + 您尚未在此頁面新增任何標記。 + + + 字型大小： + 邊距 + 行距 + 主題： + 自動 + 高對比文字 + 對齊文字 + + + 我們無法取得您的內容。 + 閱讀器偏好設定 + 筆記本 + 開啟連結 + + + 在瀏覽器中開啟 + 儲存到 Omnivore + 複製連結 + 取消 + + + 連結已儲存 + 儲存連結時出錯 + 連結已複製 + + + 儲存中 + 現在閱讀 + 稍後閱讀 + 忽略 + + + 正在儲存到 Omnivore... + 您尚未登入。請在儲存前登入。 + 頁面已儲存 + 儲存您的頁面時出錯 + + + 編輯標籤 + 封存 + 取消封存 + 分享原始內容 + 移除項目 + + + 登出 + 您確定要登出嗎？ + 確認 + 取消 + + + 管理帳戶 + 重設資料快取 + + + 設定 + + + 設定 + 文件 + 回饋 + 隱私政策 + 條款和條件 + 管理帳戶 + 登出 + diff --git a/android/Omnivore/app/src/main/res/values/strings.xml b/android/Omnivore/app/src/main/res/values/strings.xml index ad7f2a3ea..2867be8e3 100644 --- a/android/Omnivore/app/src/main/res/values/strings.xml +++ b/android/Omnivore/app/src/main/res/values/strings.xml @@ -103,6 +103,7 @@ Search Save Create a new label named \"%1$s\" + The name provided is too long (must be less or equal than %1$d characters) Labels diff --git a/apple/Omnivore.xcodeproj/project.pbxproj b/apple/Omnivore.xcodeproj/project.pbxproj index 32ee2a7a5..805222b68 100644 --- a/apple/Omnivore.xcodeproj/project.pbxproj +++ b/apple/Omnivore.xcodeproj/project.pbxproj @@ -1400,7 +1400,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.34.0; + MARKETING_VERSION = 1.35.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; @@ -1435,7 +1435,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 12.0; - MARKETING_VERSION = 1.34.0; + MARKETING_VERSION = 1.35.0; MTL_FAST_MATH = YES; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -1490,7 +1490,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.34.0; + MARKETING_VERSION = 1.35.0; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = Omnivore; PROVISIONING_PROFILE_SPECIFIER = ""; @@ -1831,7 +1831,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.34.0; + MARKETING_VERSION = 1.35.0; PRODUCT_BUNDLE_IDENTIFIER = app.omnivore.app; PRODUCT_NAME = Omnivore; PROVISIONING_PROFILE_SPECIFIER = ""; diff --git a/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift b/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift index 529fe192c..af5f05915 100644 --- a/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift +++ b/apple/OmnivoreKit/Sources/App/PDFSupport/PDFViewerViewModel.swift @@ -96,7 +96,18 @@ final class PDFViewerViewModel: ObservableObject { } } - return try await dataService.loadPDFData(slug: pdfItem.slug, pageURLString: pdfItem.originalArticleURL) + if let result = try? await dataService.loadPDFData(slug: pdfItem.slug, downloadURL: pdfItem.downloadURL) { + return result + } + + // Downloading failed, try to get the article again, and then download + if let content = try? await dataService.loadArticleContentWithRetries(itemID: pdfItem.itemID, username: "me") { + // refetched the content, now try one more time then throw + if let result = try await dataService.loadPDFData(slug: pdfItem.slug, downloadURL: content.downloadURL) { + return result + } + } + return nil } catch { print("error downloading PDF", error) return nil diff --git a/apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift b/apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift index 70ef246e1..1b7e4aa4a 100644 --- a/apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift +++ b/apple/OmnivoreKit/Sources/App/Views/LinkItemDetailView.swift @@ -46,7 +46,7 @@ import Views private func trackReadEvent() { guard let itemID = item?.unwrappedID ?? pdfItem?.itemID else { return } guard let slug = item?.unwrappedSlug ?? pdfItem?.slug else { return } - guard let originalArticleURL = item?.unwrappedPageURLString ?? pdfItem?.originalArticleURL else { return } + guard let originalArticleURL = item?.unwrappedPageURLString ?? pdfItem?.downloadURL else { return } EventTracker.track( .linkRead( diff --git a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents index 2fd575363..0351d7ee9 100644 --- a/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents +++ b/apple/OmnivoreKit/Sources/Models/CoreData/CoreDataModel.xcdatamodeld/CoreDataModel.xcdatamodel/contents @@ -1,5 +1,5 @@ - + @@ -32,6 +32,7 @@ + diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/ArticleContent.swift b/apple/OmnivoreKit/Sources/Models/DataModels/ArticleContent.swift index 2a453a9ac..4ce4eb657 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/ArticleContent.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/ArticleContent.swift @@ -16,19 +16,22 @@ public struct ArticleContent { public let highlightsJSONString: String public let contentStatus: ArticleContentStatus public let objectID: NSManagedObjectID? + public let downloadURL: String public init( title: String, htmlContent: String, highlightsJSONString: String, contentStatus: ArticleContentStatus, - objectID: NSManagedObjectID? + objectID: NSManagedObjectID?, + downloadURL: String ) { self.title = title self.htmlContent = htmlContent self.highlightsJSONString = highlightsJSONString self.contentStatus = contentStatus self.objectID = objectID + self.downloadURL = downloadURL } } diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift index 92d4eda80..b7c77bc5b 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/FeedItem.swift @@ -56,12 +56,14 @@ public struct JSONArticle: Decodable { public let isArchived: Bool public let language: String? public let wordsCount: Int? + public let downloadURL: String } public extension LinkedItem { var unwrappedID: String { id ?? "" } var unwrappedSlug: String { slug ?? "" } var unwrappedTitle: String { title ?? "" } + var unwrappedDownloadURLString: String { downloadURL ?? "" } var unwrappedPageURLString: String { pageURLString ?? "" } var unwrappedSavedAt: Date { savedAt ?? Date() } var unwrappedCreatedAt: Date { createdAt ?? Date() } diff --git a/apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift b/apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift index 43c2eb672..224e60901 100644 --- a/apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift +++ b/apple/OmnivoreKit/Sources/Models/DataModels/PDFItem.swift @@ -14,7 +14,7 @@ public struct PDFItem { public let readingProgressAnchor: Int public let isArchived: Bool public let isRead: Bool - public let originalArticleURL: String + public let downloadURL: String public let highlights: [Highlight] public static func make(item: LinkedItem) -> PDFItem? { @@ -32,7 +32,7 @@ public struct PDFItem { readingProgressAnchor: Int(item.readingProgressAnchor), isArchived: item.isArchived, isRead: item.isRead, - originalArticleURL: item.unwrappedPageURLString, + downloadURL: item.unwrappedPageURLString, highlights: item.highlights.asArray(of: Highlight.self) ) } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift index f32e5005c..a5427549b 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/ContentLoading.swift @@ -63,7 +63,8 @@ extension DataService { htmlContent: fetchResult.htmlContent, highlightsJSONString: fetchResult.highlights.asJSONString, contentStatus: fetchResult.item.isPDF ? .succeeded : fetchResult.item.state, - objectID: objectID + objectID: objectID, + downloadURL: fetchResult.item.downloadURL ) } @@ -91,7 +92,8 @@ extension DataService { .filter { $0.serverSyncStatus != ServerSyncStatus.needsDeletion.rawValue } .map { InternalHighlight.make(from: $0) }.asJSONString, contentStatus: .succeeded, - objectID: linkedItem.objectID + objectID: linkedItem.objectID, + downloadURL: linkedItem.downloadURL ?? "" ) } } @@ -166,7 +168,7 @@ extension DataService { } if articleProps.item.isPDF, needsPDFDownload { - _ = try await loadPDFData(slug: articleProps.item.slug, pageURLString: articleProps.item.pageURLString) + _ = try await loadPDFData(slug: articleProps.item.slug, downloadURL: articleProps.item.downloadURL) } try await backgroundContext.perform { [weak self] in @@ -231,7 +233,7 @@ extension DataService { _ = try await saveURL(id: id, url: url) } } catch { - // We don't propogate these errors, we just let it pass through so + // We don't propagate these errors, we just let it pass through so // the user can attempt to fetch content again. print("Error syncUnsyncedArticleContent", error) } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Public/PDFLoading.swift b/apple/OmnivoreKit/Sources/Services/DataService/Public/PDFLoading.swift index 0f4315844..8b6520640 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Public/PDFLoading.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Public/PDFLoading.swift @@ -4,8 +4,8 @@ import Models import Utils public extension DataService { - func loadPDFData(slug: String, pageURLString: String) async throws -> URL? { - guard let url = URL(string: pageURLString) else { + func loadPDFData(slug: String, downloadURL: String) async throws -> URL? { + guard let url = URL(string: downloadURL) else { throw BasicError.message(messageText: "No PDF URL found") } diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift index 56e7a77ea..a1427b35a 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/ArticleContentQuery.swift @@ -45,6 +45,7 @@ extension DataService { originalHtml: nil, language: try $0.language(), wordsCount: try $0.wordsCount(), + downloadURL: try $0.url(), recommendations: try $0.recommendations(selection: recommendationSelection.list.nullable) ?? [], labels: try $0.labels(selection: feedItemLabelSelection.list.nullable) ?? [] ), diff --git a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift index 7f897bc04..548ba02ea 100644 --- a/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift +++ b/apple/OmnivoreKit/Sources/Services/DataService/Queries/LinkedItemNetworkQuery.swift @@ -277,6 +277,7 @@ private let libraryArticleSelection = Selection.Article { originalHtml: nil, language: try $0.language(), wordsCount: try $0.wordsCount(), + downloadURL: try $0.url(), recommendations: try $0.recommendations(selection: recommendationSelection.list.nullable) ?? [], labels: try $0.labels(selection: feedItemLabelSelection.list.nullable) ?? [] ) @@ -316,6 +317,7 @@ private let searchItemSelection = Selection.SearchItem { originalHtml: nil, language: try $0.language(), wordsCount: try $0.wordsCount(), + downloadURL: try $0.url(), recommendations: try $0.recommendations(selection: recommendationSelection.list.nullable) ?? [], labels: try $0.labels(selection: feedItemLabelSelection.list.nullable) ?? [] ) diff --git a/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift b/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift index 43a7dcce9..3031cd1ef 100644 --- a/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift +++ b/apple/OmnivoreKit/Sources/Services/InternalModels/InternalLinkedItem.swift @@ -27,6 +27,7 @@ struct InternalLinkedItem { let originalHtml: String? let language: String? let wordsCount: Int? + let downloadURL: String let recommendations: [InternalRecommendation] var labels: [InternalLinkedItemLabel] @@ -65,6 +66,7 @@ struct InternalLinkedItem { linkedItem.originalHtml = originalHtml linkedItem.language = language linkedItem.wordsCount = Int64(wordsCount ?? 0) + linkedItem.downloadURL = downloadURL // Remove existing labels in case a label had been deleted if let existingLabels = linkedItem.labels { @@ -146,6 +148,7 @@ extension JSONArticle { originalHtml: nil, language: language, wordsCount: wordsCount, + downloadURL: downloadURL, recommendations: [], labels: [] ) diff --git a/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift b/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift index fff8f9b71..83350cd70 100644 --- a/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift +++ b/apple/OmnivoreKit/Sources/Views/FeedItem/LibraryItemCard.swift @@ -2,6 +2,35 @@ import Models import SwiftUI import Utils +enum FlairLabels: String { + case pinned + case favorite + case recommended + case newsletter + case rss + case feed + + var icon: Image { + switch self { + case .pinned: return Image.flairPinned + case .favorite: return Image.flairFavorite + case .recommended: return Image.flairRecommended + case .newsletter: return Image.flairNewsletter + case .feed, .rss: return Image.flairFeed + } + } + + var sortOrder: Int { + switch self { + case .feed, .rss: return 0 + case .favorite: return 1 + case .newsletter: return 2 + case .recommended: return 3 + case .pinned: return 4 + } + } +} + public extension View { func draggableItem(item: LinkedItem) -> some View { #if os(iOS) @@ -124,8 +153,30 @@ public struct LibraryItemCard: View { return "" } + var flairLabels: [FlairLabels] { + item.sortedLabels.compactMap { label in + if let name = label.name { + return FlairLabels(rawValue: name.lowercased()) + } + return nil + }.sorted { $0.sortOrder < $1.sortOrder } + } + + var nonFlairLabels: [LinkedItemLabel] { + item.sortedLabels.filter { label in + if let name = label.name, FlairLabels(rawValue: name.lowercased()) != nil { + return false + } + return true + } + } + var readInfo: some View { - AnyView(HStack { + HStack(alignment: .center, spacing: 5.0) { + ForEach(flairLabels, id: \.self) { + $0.icon + } + let fgcolor = Color.isDarkMode ? Color.themeDarkWhiteGray : Color.themeMiddleGray Text("\(estimatedReadingTime)") .font(.caption2).fontWeight(.medium) @@ -146,7 +197,7 @@ public struct LibraryItemCard: View { .font(.caption2).fontWeight(.medium) .foregroundColor(fgcolor) } - .frame(maxWidth: .infinity, alignment: .leading)) + .frame(maxWidth: .infinity, alignment: .leading) } var imageBox: some View { @@ -227,6 +278,6 @@ public struct LibraryItemCard: View { } var labels: some View { - LabelsFlowLayout(labels: item.sortedLabels) + LabelsFlowLayout(labels: nonFlairLabels) } } diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.swift b/apple/OmnivoreKit/Sources/Views/Images/Images.swift index d32ef814d..ce00165cb 100644 --- a/apple/OmnivoreKit/Sources/Views/Images/Images.swift +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.swift @@ -28,4 +28,11 @@ public extension Image { static var unarchive: Image { Image("unarchive", bundle: .module) } static var remove: Image { Image("remove", bundle: .module) } static var label: Image { Image("label", bundle: .module) } + + static var flairFeed: Image { Image("flair-feed", bundle: .module) } + static var flairFavorite: Image { Image("flair-favorite", bundle: .module) } + + static var flairNewsletter: Image { Image("flair-newsletter", bundle: .module) } + static var flairPinned: Image { Image("flair-pinned", bundle: .module) } + static var flairRecommended: Image { Image("flair-recommended", bundle: .module) } } diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Contents.json new file mode 100644 index 000000000..6738e3537 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "filename" : "Frame-2.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "filename" : "Frame-2 1.png", + "idiom" : "universal", + "scale" : "2x" + }, + { + "filename" : "Frame-2 2.png", + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 1.png new file mode 100644 index 000000000..84b84ba31 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 1.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 2.png new file mode 100644 index 000000000..049bd5920 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2 2.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2.png new file mode 100644 index 000000000..5e93251c3 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-favorite.imageset/Frame-2.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Contents.json new file mode 100644 index 000000000..04677ea52 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "filename" : "Frame.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "filename" : "Frame 1.png", + "idiom" : "universal", + "scale" : "2x" + }, + { + "filename" : "Frame 2.png", + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 1.png new file mode 100644 index 000000000..5eae617d5 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 1.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 2.png new file mode 100644 index 000000000..358758a7e Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame 2.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame.png new file mode 100644 index 000000000..3005f3c60 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-feed.imageset/Frame.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Contents.json new file mode 100644 index 000000000..8de04bec6 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "filename" : "Frame-1.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "filename" : "Frame-1 1.png", + "idiom" : "universal", + "scale" : "2x" + }, + { + "filename" : "Frame-1 2.png", + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 1.png new file mode 100644 index 000000000..dd169cb55 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 1.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 2.png new file mode 100644 index 000000000..b6d39f40c Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1 2.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1.png new file mode 100644 index 000000000..f5e9b10f9 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-newsletter.imageset/Frame-1.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Contents.json new file mode 100644 index 000000000..a55c67c87 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "filename" : "Frame-3.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "filename" : "Frame-3 1.png", + "idiom" : "universal", + "scale" : "2x" + }, + { + "filename" : "Frame-3 2.png", + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 1.png new file mode 100644 index 000000000..dba4109e8 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 1.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 2.png new file mode 100644 index 000000000..487a9b273 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3 2.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3.png new file mode 100644 index 000000000..1d0df2685 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-pinned.imageset/Frame-3.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Contents.json b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Contents.json new file mode 100644 index 000000000..239db47c1 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Contents.json @@ -0,0 +1,23 @@ +{ + "images" : [ + { + "filename" : "Frame-4.png", + "idiom" : "universal", + "scale" : "1x" + }, + { + "filename" : "Frame-4 1.png", + "idiom" : "universal", + "scale" : "2x" + }, + { + "filename" : "Frame-4 2.png", + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "author" : "xcode", + "version" : 1 + } +} diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 1.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 1.png new file mode 100644 index 000000000..a1ffdd088 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 1.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 2.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 2.png new file mode 100644 index 000000000..3892567f1 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4 2.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4.png b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4.png new file mode 100644 index 000000000..c2f5d96f3 Binary files /dev/null and b/apple/OmnivoreKit/Sources/Views/Images/Images.xcassets/flair-recommended.imageset/Frame-4.png differ diff --git a/apple/OmnivoreKit/Sources/Views/Resources/zh-Hant.lproj/Localizable.strings b/apple/OmnivoreKit/Sources/Views/Resources/zh-Hant.lproj/Localizable.strings new file mode 100644 index 000000000..a9dafd494 --- /dev/null +++ b/apple/OmnivoreKit/Sources/Views/Resources/zh-Hant.lproj/Localizable.strings @@ -0,0 +1,206 @@ +// Unit test Entry -- Do not remove this or add entries before this one. +// This allows us to check for syntax errors in this file with a unit test +"unitTestLeadingEntry" = "僅供測試用途。"; + +// share extension +"saveArticleSavedState" = "已儲存至 Omnivore"; +"saveArticleProcessingState" = "正在儲存至 Omnivore"; +"extensionAppUnauthorized" = "請先從應用程式登入 Omnivore，然後再儲存您的第一個連結。"; +"saveToOmnivore" = "儲存至 Omnivore"; + +// audio player +"audioPlayerReplay" = "重新播放"; + +// Highlights List Card +"highlightCardHighlightByOther" = "由其他人標註"; +"highlightCardNoHighlightsOnPage" = "您尚未在此頁面新增任何標註。"; + +// Labels View +"labelsViewAssignNameColor" = "指定名稱和顏色。"; +"createLabelMessage" = "建立新標籤"; +"labelsPurposeDescription" = "使用標籤來建立連結的精選集。"; +"labelNamePlaceholder" = "標籤名稱"; + +// Manage Account View +"manageAccountDelete" = "刪除帳戶"; +"manageAccountResetCache" = "重設資料快取"; +"manageAccountConfirmDeleteMessage" = "您確定要刪除您的帳戶嗎？此動作無法撤銷。"; + +// Newsletter Emails View +"newsletterEmailsExisting" = "現有電子郵件（點選以複製）"; +"createNewEmailMessage" = "建立新的電子郵件地址"; +"newslettersDescription" = "將 PDF 加入您的圖書館，或使用 Omnivore 電子郵件地址訂閱電子報。"; +"noCurrentSubscriptionsMessage" = "您目前沒有任何訂閱。"; + +// Profile View +"profileConfirmLogoutMessage" = "您確定要登出嗎？"; + +// Devices View +"devicesTokensTitle" = "已註冊的裝置權杖（滑動以移除）"; +"devicesCreated" = "已建立："; + +// Push Notification Settings +"notificationsEnabled" = "已啟用通知"; +"notificationsExplainer" = "啟用推播通知將授予 Omnivore 裝置傳送通知的權限，\n但您可以控制哪些通知被傳送。"; +"notificationsTriggerExplainer" = "推播通知是由您的\n[帳戶規則](https://omnivore.app/settings/rules) 觸發，您可以在線上編輯。"; +"notificationsEnable" = "啟用推播通知？"; +"notificationsGeneralExplainer" = "當電子報連結抵達您的收件匣時獲得通知。或從我們的分享擴充功能接收提醒。"; +"notificationsOptionDeny" = "不，謝謝"; +"notificationsOptionEnable" = "是的，請繼續"; + +// Community Modal +"communityHeadline" = "協助建立 Omnivore 社群"; +"communityAppstoreReview" = "在 AppStore 上評價"; +"communityTweet" = "關於 Omnivore 的推文"; +"communityFollowTwitter" = "在 Twitter 上追蹤我們"; +"communityJoinDiscord" = "在 Discord 上加入我們"; +"communityStarGithub" = "在 GitHub 上給我們星星"; + +// Clubs View +"clubsLearnTitle" = "了解更多關於俱樂部的資訊"; +"clubsName" = "俱樂部名稱"; +"clubsCreate" = "建立新俱樂部"; +"clubsYours" = "您的俱樂部"; +"clubsNotAMemberMessage" = "您並非任何俱樂部的成員。\n建立新俱樂部並將邀請連結傳送給您的朋友以開始。\n\n在測試版期間，您最多可以建立三個俱樂部，每個俱樂部\n可以有最多十二個使用者。"; +"clubsErrorCopying" = "複製邀請 URL 時發生錯誤"; +"clubsAdminDenyViewing" = "此俱樂部的管理員不允許檢視所有成員。"; +"clubsNoMembers" = "此俱樂部沒有任何成員。透過傳送\n邀請連結將使用者加入您的俱樂部。"; +"clubsLeave" = "離開俱樂部"; +"clubsLeaveConfirm" = "您確定要離開此俱樂部嗎？不會刪除任何資料，但您將停止接收來自俱樂部的推薦。"; +"clubsNoneJoined" = "您沒有可以發布的俱樂部。\n加入俱樂部或建立自己的俱樂部以開始推薦文章。"; + +// Subscriptions +"subscriptionsErrorRetrieving" = "抱歉，我們無法取得您的訂閱。"; +"subscriptionsNone" = "您目前沒有任何訂閱。"; +//"subscriptions.error.retrieving" = "Last received: \(updatedDate.formatted())"; // unused for now + +// Text to Speech +"texttospeechLanguageDefault" = "預設語言"; +"texttospeechSettingsAudio" = "音訊設定"; +"texttospeechSettingsEnablePrefetch" = "啟用音訊預取"; +"texttospeechBetaSignupInProcess" = "正在報名測試版"; +"texttospeechBetaRealisticVoiceLimit" = "您正在參加超逼真語音測試版。在測試版期間，您每天可以聆聽 10,000 個單詞的音訊。"; +"texttospeechBetaRequestReceived" = "我們已收到您參加超逼真語音示範的請求。當有空位時，我們將透過電子郵件通知您。"; +"texttospeechBetaWaitlist" = "超逼真語音目前處於有限的測試版，並僅提供英語。啟用此功能將會將您加入測試版佇列。"; + +// Sign in/up +"registrationNoAccount" = "還沒有帳戶？"; +"registrationForgotPassword" = "忘記密碼？"; +"registrationStatusCheck" = "檢查狀態"; +"registrationUseDifferentEmail" = "使用不同的電子郵件？"; +"registrationFullName" = "全名"; +"registrationUsername" = "使用者名稱"; +"registrationAlreadyHaveAccount" = "已經有帳戶？"; +"registrationBio" = "個人簡介（選填）"; +"registrationWelcome" = "歡迎來到 Omnivore！"; +"registrationUsernameAssignedPrefix" = "您的使用者名稱是："; +"registrationChangeUsername" = "更改使用者名稱"; +"registrationEdit" = "編輯"; +"googleAuthButton" = "使用 Google 繼續"; +"registrationViewSignUpHeadline" = "註冊"; +"loginErrorInvalidCreds" = "提供的登入憑證無效。"; + +// Recommendation +"recommendationToPrefix" = "至："; +"recommendationAddNote" = "新增註解（選填）"; +//"recommendationToPrefix" = "Include your \(viewModel.highlightCount) highlight\(viewModel.highlightCount > 1 ? "s" : """; // unused for now +"recommendationError" = "推薦此頁面時發生錯誤"; + +// Web Reader +"readerCopyLink" = "複製連結"; +"readerSave" = "儲存至 Omnivore"; +"readerError" = "發生錯誤"; + +// Debug Menu +"menuDebugTitle" = "除錯選單"; +"menuDebugApiEnv" = "API 環境："; + +// Navigation +"navigationSelectLink" = "從您的圖書館選擇一個連結"; +"navigationSelectSidebarToggle" = "切換側邊欄"; + +// Welcome View +"welcomeTitle" = "專為認真讀者設計的稍後閱讀工具。"; +"welcomeLearnMore" = "了解更多"; +"welcomeSignupAgreement" = "註冊即表示您同意 Omnivore 的\n"; +"welcomeTitleTermsOfService" = "服務條款"; +"welcomeTitleAndJoiner" = " 和 "; +"welcomeTitleEmailContinue" = "使用電子郵件繼續"; + +// Keyboard Commands +"keyboardCommandDecreaseFont" = "減小字型大小"; +"keyboardCommandIncreaseFont" = "增大字型大小"; +"keyboardCommandDecreaseMargin" = "減小邊距"; +"keyboardCommandIncreaseMargin" = "增大邊距"; +"keyboardCommandDecreaseLineSpacing" = "減小行距"; +"keyboardCommandIncreaseLineSpacing" = "增大行距"; + +// Library +//"library.by.author.suffix" = "by \(author)" // unused +//"Recommended by \(byStr) in \(inStr)" // unused + +// Generic +"genericSnooze" = "稍後提醒"; +"genericClose" = "關閉"; +"genericCreate" = "建立"; +"genericConfirm" = "確認"; +"genericProfile" = "個人資料"; +"genericNext" = "下一步"; +"genericName" = "名稱"; +"genericOk" = "確定"; +"genericRetry" = "重試"; +"genericEmail" = "電子郵件"; +"genericPassword" = "密碼"; +"genericSubmit" = "送出"; +"genericContinue" = "繼續"; +"genericSend" = "傳送"; +"genericOptions" = "選項"; +"genericOpen" = "開啟"; +"genericChangeApply" = "套用變更"; +"genericTitle" = "標題"; +"genericAuthor" = "作者"; +"genericDescription" = "描述"; +"genericSave" = "儲存"; +"genericLoading" = "載入中..."; +"genericFontFamily" = "字型家族"; +"genericHighContrastText" = "高對比文字"; +"enableHighlightOnReleaseText" = "自動標註模式"; +"enableJustifyText" = "對齊文字"; +"genericFont" = "字型"; +"genericHighlight" = "標註"; +"labelsGeneric" = "標籤"; +"emailsGeneric" = "電子郵件"; +"subscriptionsGeneric" = "訂閱"; +"textToSpeechGeneric" = "文字轉語音"; +"privacyPolicyGeneric" = "隱私權政策"; +"termsAndConditionsGeneric" = "使用條款"; +"feedbackGeneric" = "意見回饋"; +"manageAccountGeneric" = "管理帳戶"; +"logoutGeneric" = "登出"; +"doneGeneric" = "完成"; +"cancelGeneric" = "取消"; +"exportGeneric" = "匯出"; +"inboxGeneric" = "收件匣"; +"readLaterGeneric" = "稍後閱讀"; +"newslettersGeneric" = "電子報"; +"allGeneric" = "全部"; +"archivedGeneric" = "已封存"; +"highlightedGeneric" = "已標註"; +"filesGeneric" = "檔案"; +"newestGeneric" = "最新"; +"oldestGeneric" = "最舊"; +"recentlyReadGeneric" = "最近閱讀"; +"recentlyPublishedGeneric" = "最近發布"; +"clubsGeneric" = "俱樂部"; +"filterGeneric" = "篩選器"; +"errorGeneric" = "發生錯誤，請再試一次。"; +"pushNotificationsGeneric" = "推播通知"; +"dismissButton" = "關閉"; +"errorNetwork" = "我們無法連線到網路。"; +"documentationGeneric" = "文件"; + +// TODO: search navigationTitle, toggle, section, button, Label, title: ", CreateProfileViewModel, TextField, .keyboardShortcut + +// Unit test Entry -- Do not remove this or add entries after this one. +// This allows us to check for syntax errors in this file with a unit test +"unitTestTrailingEntry" = "僅供測試用途。"; diff --git a/apple/gql-server/package.json b/apple/gql-server/package.json index 3337f3b4b..9088164b4 100644 --- a/apple/gql-server/package.json +++ b/apple/gql-server/package.json @@ -9,6 +9,6 @@ "dependencies": { "express": "^4.18.1", "express-graphql": "^0.12.0", - "graphql": "^16.4.0" + "graphql": "^16.8.1" } } \ No newline at end of file diff --git a/apple/gql-server/yarn.lock b/apple/gql-server/yarn.lock index ae72deb0f..587c08d1d 100644 --- a/apple/gql-server/yarn.lock +++ b/apple/gql-server/yarn.lock @@ -194,10 +194,10 @@ get-intrinsic@^1.0.2: has "^1.0.3" has-symbols "^1.0.3" -graphql@^16.4.0: - version "16.6.0" - resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.6.0.tgz#c2dcffa4649db149f6282af726c8c83f1c7c5fdb" - integrity sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw== +graphql@^16.8.1: + version "16.8.1" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.8.1.tgz#1930a965bef1170603702acdb68aedd3f3cf6f07" + integrity sha512-59LZHPdGZVh695Ud9lRzPBVTtlX9ZCV150Er2W43ro37wVof0ctenSaskPPjN7lVTIN8mSZt8PHUNKZuNQUuxw== has-symbols@^1.0.3: version "1.0.3" diff --git a/docs/guides/getting-started/ar/getting-started-guide.md b/docs/guides/getting-started/ar/getting-started-guide.md new file mode 100644 index 000000000..ebf411e68 --- /dev/null +++ b/docs/guides/getting-started/ar/getting-started-guide.md @@ -0,0 +1,178 @@ + +# الشروع في إستعمال أمنيفور + +تطبيقة أومنيفور هي تطبيقة للقراءة لاحقاً تتيح لك من تسجيل وتنظيم كل ما تقرأه على الإنترنت + +هذا الدليل سيوضح لك كيفية استخدام الوظائف الأساسية والميزات المتقدمة في أومنيفور، مقسمة إلى أربعة نشاطات رئيسية + +- حفض +- قراءة +- تنظيم +- اندماجات + + +المكتبة هي مركز تجربتك في أومنيفور، حيث يمكنك الوصول بسرعة إلى أي روابط قمت بحفظها. +الروابط المحفوظة تبقى في مكتبتك إلى الأبد ما لم تقم بحذفها. + +## حفظ + +هناك خمس طرق رئيسية لحفظ الروابط للصفحات أو المقالات التي ترغب في قراءتها لاحقًا: + +1 - الحفظ من مكتبة أومنيفور الخاصة بك +2 - الحفظ من متصفح +3 - الحفظ من هاتف أو جهاز لوحي (آي أو إس أو أندرويد) +4 - الاشتراك في النشرات الإخبارية عبر البريد الإلكتروني +5 - حفظ ملفات بي دي إف من جهاز ماك + + +### الحفض من مكتبة أمنيفور + +1 - في الزاوية العليا اليمنى من مكتبتك، انقر على زر إضافة رابط. +2- أدخل العنوان الذي ترغب في حفظه وانقر على إضافة رابط. +3 - سيظهر الرابط في مكتبتك في المرة التالية التي تقوم فيها بتحديثها. + + +### الحفظ من المتصفح + +1 - قم بتحميل وتثبيت إضافة أومنيفور لمتصفحك: + +- [كروم ](https://omnivore.app/install/chrome) +- [ايدج](https://omnivore.app/install/edge) +- [فاير فوكس](https://omnivore.app/install/firefox) +- [سفري](https://omnivore.app/install/safari) + +2 -انتقل إلى الصفحة التي ترغب في حفظها وانقر على زر أومنيفور في شريط أدوات المتصفح أو قائمة الإضافات. +3 - أو بدلاً عن ذلك يمكنك النقر بزر فأرة الحاسوب الأيمن على أي رابط تشعبي أو اختر **حفظ إلى أمنيفور** من القائمة. +4 - سيظهر الرابط في مكتبتك في المرة التالية التي تقوم فيها بتحديثها. + + +### الحفظ من الهاتف أو الجهاز اللوحي + +أفضل طريقة لحفظ الروابط من جهازك المحمول هي عبر تطبيق أمنيفور. يمكنك تحميل التطبيق هنا: + + +أي أو إس (ايباد أو أيفون) (https://omnivore.app/install/ios) +أندرويد + + +بعد تثبيت تطبيق الهاتف المحمول: + +1 - في المتصفح الخاص بك، انتقل إلى الصفحة التي ترغب في حفظها واضغط على زر شارك +2 - اضغط على أيقونة أومنيفور في قائمة الشارك +3 - سيظهر الرابط في مكتبتك عندما تقوم بتحديثها في المرة القادمة + + +### الاشتراكات في النشرة الإخبارية عبر البريد الإلكتروني + +1 - على موقع أو تطبيق أومنيفور، اضغط على صورتك أو الحروف الأولى أو الصورة الرمزية في الزاوية العليا اليمنى للوصول إلى قائمة الملف الشخصي. + اختر البريد الإلكتروني من القائمة. + 2 - اضغط على إنشاء عنوان بريد إلكتروني جديد لإضافة عنوان بريد إلكتروني جديد (مثل: username-123_abc@inbox.omnivore.app) إلى القائمة. + 3 - انقر على أيقونة النسخ بجوار عنوان البريد الإلكتروني + 4 - انتقل إلى صفحة التسجيل للنشرةgetting-started-guide.md الإخبارية التي ترغب في الاشتراك فيها + 5 - الصق عنوان البريد الإلكتروني أومنيفور في نموذج التسجيل + 6 - سيتم توصيل النشرات الإخبارية الجديدة تلقائيًا إلى صندوق الوارد أومنيفور الخاص بك + + +### حفظ ملفات البي دي إيف من جهاز الماك + +1 - قم بتثبيت تطبيقة الماك(https://omnivore.app/install/mac) +2 - قم بتحديد موقع ملف البي دي إف الذي ترغب في حفظه ثم انقر على إسم الملف بالزر الايمن من فأرة الحاسوب +3 - اختر شارك من القائمة واختر أومنيفور +4 - سيظهر الرابط في مكتبتك عندما تقوم بتحديثها في المرة القادمة + + +## القراءة + +انقر على أي رابط محفوظ في مكتبتك لدخول وضع القراءة +تقوم أومنيفور بتنسيق الصفحات لتسهيل القراءة والتظليل، وإزالة الإعلانات والتلهية لقراءة خالية من التشتيت. كما يجعل وضع التركيز على نص المقالات أصغر حجمًا وأسرع في التحميل. +أثناء القراءة، يمكنك: +- تغيير التنسيق +- تسليط الضوء على النص +- إضافة ملاحظات +- عرض كافة النقاط البارزة والملاحظات المحفوظة +- تتبع تقدم القراءة + +### تغيير التنسيق + +1 - **_المظهر_**: اضغط على صورتك، أو الحروف الأولى، أو الصورة الرمزية في ا +لزاوية العليا اليمنى للوصول إلى قائمة الملف الشخصي. اختر الصورة المصغرة البيضاء أو السوداء لاختيار مظهر فاتح أو مظهر داكن + +2 - **_ تنسيق النص_** : اضغط على ايقونة Aa لتعديل حجم النص، الخط،الهوامش وتباعد الأسطر + + +### تسليط الضوء على النص + +1 - حدد النص الذي ترغب في تسليط الضوء عليه +2 - اضغط على زر تسليط الضوء +3 - سيظهر النص مظللاً عندما تعاود قراءة المقال في المرة القادمة + + +### إضافة ملاحظات + +1 - قم بتظليل قسم من النص حيث ترغب في إضافة ملاحظة +2 - اضغط على زر ملاحظة، اكتب ملاحظتك، ثم اضغط على حفظ +3 - ستظهر أيقونة الملاحظة عندما تعيد قراءة هذا المقال في المرة القادمة + +### عرض جميع النصوص المظللة والملاحظات المحفوظة +1 - اضغط على أيقونة تظليل/ملاحظة لرؤية قائمة بجميع النصوص المظللة والملاحظات التي أضفتها إلى هذه الصفحة. +2 - لإزالة ملاحظة أو تظليل، اخترها من القائمة ثم اضغط على أيقونة السلة + + + ### تتبع تقدم القراءة + + يقوم أومنيفور بتتبع تقدم القراءة الخاص بك تلقائيًا + عبر أجهزتك المختلفة بحيث يمكنك استئناف القراءة من حيث توقفت بسهولة. سيظهر شريط التقدم في أعلى كل رابط في مكتبتك بعد بدء القراءة. + + +## تنظيم + +بشكل تلقائي، يعرض صندوق الوارد في المكتبة جميع الروابط التي قد قمت بحفظها. لإدارة قائمتك والحفاظ على تنظيم قراءتك، يوفر أمنيفور الإجراءات التالية: + +- الأرشفة +- التسميات +- البحث +- الفلاتر + +### الأرشفة + +1 - اضغط على أيقونة القائمة بجوار الرابط الذي ترغب في أرشفته (على تطبيق الجوال، اضغط مع الاستمرار على الرابط لفتح القائمة) +2 - اختر **أرشفة** +3 - الرابط سيختفي من المكتبة التلقائية لكن سيضهر إذا اخترت فلتر الارشفة(إنظر إلى الفلاتر ) + +### التسميات + +1 - اضغط على أيقونة القائمة بجوار أي رابط واختر تعيين التسميات. +2 - اختر تسمية موجودة من القائمة أو اضغط على تعديل التسميات لإنشاء تسمية جديدة +3 - ستظهر التسمية بجوار الرابط في مكتبتك. اضغط عليها لعرض جميع الروابط التي تحمل نفس التسمية +4 - على الهاتف المحمول فقط: اضغط على التسميات لرؤية قائمة جميع التسميات التي استخدمتها . اضغط على واحدة لعرض جميع الروابط التي تحمل نفس التسمية +5 - ملاحظة : أمنيفور سيقوم بتعيين بعض التسميات اتوماتيكيا مثل "Newsletters" + +### البحث + +1 - للبحث في جميع الروابط التي قمت بحفظها، أدخل كلمة أو عبارة في شريط البحث +2 - يمكنك دمج الكلمات الرئيسية مع التسميات والفلاتر لتركيز بحثك بشكل أكبر[تعرف اكثر على البحث المتقدم ](https://docs.omnivore.app/using/searcl) + + +### الفيلتر + +1 - استخدم قائمة **فلتر** لتحسين عرض المكتبة الخاصة بك (قد تكون بعض فلتر مرئية بشكل طبيعي) +2 - اختر **القراءة لاحقاً** لعرض قائمة بجميع الروابط الغير مؤرشفة باستثناء النشرات الإخبارية +3 - اختر **التمييزات** لعرض النصوص التي قمت بتمييزها في جميع الصفحات المحفوظة +4 - اختر **هذا اليوم ** لعرض قائمة الروابط التي قمت بحفظها اليوم +5 - اختر **النشرات الإخبارية** لعرض الروابط المحفوظة عبر اشتراكاتك في النشرات الإخبارية + + +### الاندماجات +أمنيفور يتيح لك إستعمال اندماجات بقواعد المعرفة وتطبيقات تدوين الملاحظات مثل: +- لغس سيك +- ويب هوكس + +### لغس سيك +بفضل وصلة لغس سيك، يمكنك مزامنة جميع المقالات والتمييزات والملاحظات إلى لغس سيك ، قاعدة معرفة شهيرة. +للحصول على معلومات حول كيفية إعداد واستخدام لغس سيك يرجى الرجوع لهذا الدليل [Omnivore for Logseq Plugin Guide](https://briansunter.com/graph/#/page/omnivore-logseq-guide) + + +### ويب هوكس +يمكن لي أمنيفور تشغيل الويب هوكس عندما تحفض رابط تقروه + مثال : This example يريك إستعمال ويب هوكس لكتابة كل الروابط المحفوظة على جوجل شيتس المسجلة على جوجل درايف + diff --git a/package.json b/package.json index de5c1658e..55b813a4b 100644 --- a/package.json +++ b/package.json @@ -40,4 +40,4 @@ "yarn": "1.22.19" }, "dependencies": {} -} +} \ No newline at end of file diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 416c65b68..07f4ccd05 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -129,6 +129,10 @@ export class LibraryItem { @JoinColumn({ name: 'upload_file_id' }) uploadFile?: UploadFile + // get upload_file_id without joining relations + @Column('text', { nullable: true }) + uploadFileId?: string + @Column('enum', { enum: ContentReaderType, default: ContentReaderType.WEB }) contentReader!: ContentReaderType diff --git a/packages/api/src/entity/subscription.ts b/packages/api/src/entity/subscription.ts index 90f7d593b..fea63d890 100644 --- a/packages/api/src/entity/subscription.ts +++ b/packages/api/src/entity/subscription.ts @@ -59,9 +59,15 @@ export class Subscription { @Column('timestamp', { nullable: true }) lastFetchedAt?: Date | null + @Column('text', { nullable: true }) + lastFetchedChecksum?: string | null + @CreateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) createdAt!: Date @UpdateDateColumn({ default: () => 'CURRENT_TIMESTAMP' }) updatedAt!: Date + + @Column('timestamp', { nullable: true }) + scheduledAt?: Date | null } diff --git a/packages/api/src/entity/user.ts b/packages/api/src/entity/user.ts index a926b5a89..83526f946 100644 --- a/packages/api/src/entity/user.ts +++ b/packages/api/src/entity/user.ts @@ -22,6 +22,7 @@ export enum RegistrationType { export enum StatusType { Active = 'ACTIVE', Pending = 'PENDING', + Deleted = 'DELETED', } @Entity() diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index f0f41ff68..da68dbe22 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -2092,6 +2092,7 @@ export enum SaveArticleReadingProgressErrorCode { } export type SaveArticleReadingProgressInput = { + force?: InputMaybe; id: Scalars['ID']; readingProgressAnchorIndex?: InputMaybe; readingProgressPercent: Scalars['Float']; @@ -2616,9 +2617,8 @@ export enum SubscribeErrorCode { } export type SubscribeInput = { - name?: InputMaybe; subscriptionType?: InputMaybe; - url?: InputMaybe; + url: Scalars['String']; }; export type SubscribeResult = SubscribeError | SubscribeSuccess; @@ -2978,7 +2978,9 @@ export type UpdateSubscriptionInput = { description?: InputMaybe; id: Scalars['ID']; lastFetchedAt?: InputMaybe; + lastFetchedChecksum?: InputMaybe; name?: InputMaybe; + scheduledAt?: InputMaybe; status?: InputMaybe; }; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index 3ef33f886..dec3e34f6 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -1571,6 +1571,7 @@ enum SaveArticleReadingProgressErrorCode { } input SaveArticleReadingProgressInput { + force: Boolean id: ID! readingProgressAnchorIndex: Int readingProgressPercent: Float! @@ -2058,9 +2059,8 @@ enum SubscribeErrorCode { } input SubscribeInput { - name: String subscriptionType: SubscriptionType - url: String + url: String! } union SubscribeResult = SubscribeError | SubscribeSuccess @@ -2391,7 +2391,9 @@ input UpdateSubscriptionInput { description: String id: ID! lastFetchedAt: Date + lastFetchedChecksum: String name: String + scheduledAt: Date status: SubscriptionStatus } diff --git a/packages/api/src/repository/library_item.ts b/packages/api/src/repository/library_item.ts index 1a428d65b..1e7e3d708 100644 --- a/packages/api/src/repository/library_item.ts +++ b/packages/api/src/repository/library_item.ts @@ -17,4 +17,46 @@ export const libraryItemRepository = appDataSource countByCreatedAt(createdAt: Date) { return this.countBy({ createdAt }) }, + + createByPopularRead(name: string, userId: string) { + return this.query( + ` + INSERT INTO omnivore.library_item ( + slug, + readable_content, + original_content, + description, + title, + author, + original_url, + item_type, + thumbnail, + published_at, + site_name, + user_id, + word_count + ) + SELECT + slug, + readable_content, + original_content, + description, + title, + author, + original_url, + $1, + thumbnail, + published_at, + site_name, + $2, + word_count + FROM + omnivore.popular_read + WHERE + key = $3 + RETURNING * + `, + ['ARTICLE', userId, name] + ) as Promise + }, }) diff --git a/packages/api/src/repository/user.ts b/packages/api/src/repository/user.ts index 408543c95..a8f327b0d 100644 --- a/packages/api/src/repository/user.ts +++ b/packages/api/src/repository/user.ts @@ -1,6 +1,6 @@ import { In } from 'typeorm' import { appDataSource } from '../data_source' -import { User } from './../entity/user' +import { StatusType, User } from './../entity/user' const TOP_USERS = [ 'jacksonh', @@ -16,7 +16,7 @@ export const MAX_RECORDS_LIMIT = 1000 export const userRepository = appDataSource.getRepository(User).extend({ findById(id: string) { - return this.findOneBy({ id }) + return this.findOneBy({ id, status: StatusType.Active }) }, findByEmail(email: string) { diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index bf26872c5..83ea4830b 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -46,6 +46,7 @@ import { TypeaheadSearchSuccess, UpdateReason, UpdatesSinceError, + UpdatesSinceErrorCode, UpdatesSinceSuccess, } from '../../generated/graphql' import { getColumns } from '../../repository' @@ -53,7 +54,6 @@ import { getInternalLabelWithColor } from '../../repository/label' import { libraryItemRepository } from '../../repository/library_item' import { userRepository } from '../../repository/user' import { createPageSaveRequest } from '../../services/create_page_save_request' -import { findHighlightsByLibraryItemId } from '../../services/highlights' import { addLabelsToLibraryItem, findLabelsByIds, @@ -62,11 +62,11 @@ import { } from '../../services/labels' import { createLibraryItem, - findLibraryItemById, findLibraryItemByUrl, findLibraryItemsByPrefix, searchLibraryItems, updateLibraryItem, + updateLibraryItemReadingProgress, updateLibraryItems, } from '../../services/library_item' import { parsedContentToLibraryItem } from '../../services/save_page' @@ -82,14 +82,12 @@ import { cleanUrl, errorHandler, generateSlug, - isBase64Image, isParsingTimeout, libraryItemToArticle, libraryItemToSearchItem, titleForFilePath, userDataToUser, } from '../../utils/helpers' -import { createImageProxyUrl } from '../../utils/imageproxy' import { contentConverter, getDistillerResult, @@ -98,10 +96,7 @@ import { parsePreparedContent, } from '../../utils/parser' import { parseSearchQuery, sortParamsToSort } from '../../utils/search' -import { - getStorageFileDetails, - makeStorageFilePublic, -} from '../../utils/uploads' +import { getStorageFileDetails } from '../../utils/uploads' import { itemTypeForContentType } from '../upload_files' export enum ArticleFormat { @@ -310,7 +305,6 @@ export const createArticleResolver = authorized< pubsub ) } - await makeStorageFilePublic(uploadFileData.id, uploadFileData.fileName) } let libraryItemToReturn: LibraryItem @@ -570,16 +564,11 @@ export const saveArticleReadingProgressResolver = authorized< readingProgressPercent, readingProgressAnchorIndex, readingProgressTopPercent, + force, }, }, - { uid, pubsub } + { log, pubsub, uid } ) => { - const libraryItem = await findLibraryItemById(id, uid) - - if (!libraryItem) { - return { errorCodes: [SaveArticleReadingProgressErrorCode.NotFound] } - } - if ( readingProgressPercent < 0 || readingProgressPercent > 100 || @@ -590,40 +579,47 @@ export const saveArticleReadingProgressResolver = authorized< ) { return { errorCodes: [SaveArticleReadingProgressErrorCode.BadData] } } - // If we have a top percent, we only save it if it's greater than the current top percent - // or set to zero if the top percent is zero. - const readingProgressTopPercentToSave = readingProgressTopPercent - ? Math.max( - readingProgressTopPercent, - libraryItem.readingProgressTopPercent || 0 + try { + if (force) { + // update reading progress without checking the current value + const updatedItem = await updateLibraryItem( + id, + { + readingProgressBottomPercent: readingProgressPercent, + readingProgressTopPercent: readingProgressTopPercent ?? undefined, + readingProgressHighestReadAnchor: + readingProgressAnchorIndex ?? undefined, + readAt: new Date(), + }, + uid, + pubsub ) - : readingProgressTopPercent === 0 - ? 0 - : undefined - // If setting to zero we accept the update, otherwise we require it - // be greater than the current reading progress. - const updatedPart: QueryDeepPartialEntity = { - readingProgressBottomPercent: - readingProgressPercent === 0 - ? 0 - : Math.max( - readingProgressPercent, - libraryItem.readingProgressBottomPercent - ), - readingProgressHighestReadAnchor: - readingProgressAnchorIndex === 0 - ? 0 - : Math.max( - readingProgressAnchorIndex || 0, - libraryItem.readingProgressHighestReadAnchor - ), - readingProgressTopPercent: readingProgressTopPercentToSave, - readAt: new Date(), - } - const updatedItem = await updateLibraryItem(id, updatedPart, uid, pubsub) - return { - updatedArticle: libraryItemToArticle(updatedItem), + return { + updatedArticle: libraryItemToArticle(updatedItem), + } + } + + // update reading progress only if the current value is lower + const updatedItem = await updateLibraryItemReadingProgress( + id, + uid, + readingProgressPercent, + readingProgressTopPercent, + readingProgressAnchorIndex, + pubsub + ) + if (!updatedItem) { + return { errorCodes: [SaveArticleReadingProgressErrorCode.BadData] } + } + + return { + updatedArticle: libraryItemToArticle(updatedItem), + } + } catch (error) { + log.error('saveArticleReadingProgressResolver error', error) + + return { errorCodes: [SaveArticleReadingProgressErrorCode.Unauthorized] } } } ) @@ -634,7 +630,7 @@ export const searchResolver = authorized< QuerySearchArgs >(async (_obj, params, { log, uid }) => { const startCursor = params.after || '' - const first = params.first || 10 + const first = Math.min(params.first || 10, 100) // limit to 100 items // the query size is limited to 255 characters if (params.query && params.query.length > 255) { @@ -649,7 +645,7 @@ export const searchResolver = authorized< size: first + 1, // fetch one more item to get next cursor sort: searchQuery.sort, includePending: true, - includeContent: params.includeContent || false, + includeContent: !!params.includeContent, ...searchQuery, }, uid @@ -665,25 +661,7 @@ export const searchResolver = authorized< libraryItems.pop() } - await Promise.all( - libraryItems.map(async (libraryItem) => { - if ( - libraryItem.highlightAnnotations && - libraryItem.highlightAnnotations.length > 0 - ) { - // fetch highlights for each item - libraryItem.highlights = await findHighlightsByLibraryItemId( - libraryItem.id, - uid - ) - } - }) - ) - const edges = libraryItems.map((libraryItem) => { - if (libraryItem.siteIcon && !isBase64Image(libraryItem.siteIcon)) { - libraryItem.siteIcon = createImageProxyUrl(libraryItem.siteIcon, 128, 128) - } if (params.includeContent && libraryItem.readableContent) { // convert html to the requested format const format = params.format || ArticleFormat.Html @@ -747,7 +725,12 @@ export const updatesSinceResolver = authorized< const startCursor = after || '' const size = first || 10 - const startDate = new Date(since) + let startDate = new Date(since) + if (isNaN(startDate.getTime())) { + // for android app compatibility + startDate = new Date(0) + } + const { libraryItems, count } = await searchLibraryItems( { from: Number(startCursor), diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index c2cc34f9a..ca145815e 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -6,15 +6,19 @@ import { Subscription } from '../entity/subscription' import { Article, + Highlight, Label, PageType, Recommendation, SearchItem, } from '../generated/graphql' +import { findHighlightsByLibraryItemId } from '../services/highlights' import { findLabelsByLibraryItemId } from '../services/labels' import { findRecommendationsByLibraryItemId } from '../services/recommendation' import { findUploadFileById } from '../services/upload_file' import { + highlightDataToHighlight, + isBase64Image, recommandationDataToRecommendation, validatedDate, wordsCount, @@ -461,7 +465,7 @@ export const functionResolvers = { async url(item: SearchItem, _: unknown, ctx: WithDataSourcesContext) { if ( (item.pageType == PageType.File || item.pageType == PageType.Book) && - ctx.uid && + ctx.claims && item.uploadFileId ) { const upload = await findUploadFileById(item.uploadFileId) @@ -483,30 +487,64 @@ export const functionResolvers = { if (item.wordCount) return item.wordCount return item.content ? wordsCount(item.content) : undefined }, + siteIcon(item: { siteIcon?: string }) { + if (item.siteIcon && !isBase64Image(item.siteIcon)) { + return createImageProxyUrl(item.siteIcon, 128, 128) + } + + return item.siteIcon + }, + async highlights( + item: { + id: string + highlights?: Highlight[] + highlightAnnotations?: string[] | null + }, + _: unknown, + ctx: WithDataSourcesContext + ) { + if (item.highlights) return item.highlights + + if (item.highlightAnnotations && item.highlightAnnotations.length > 0) { + const highlights = await findHighlightsByLibraryItemId(item.id, ctx.uid) + return highlights.map(highlightDataToHighlight) + } + + return [] + }, async labels( - item: { id: string; labels?: Label[] }, + item: { id: string; labels?: Label[]; labelNames?: string[] | null }, _: unknown, ctx: WithDataSourcesContext ) { if (item.labels) return item.labels - return findLabelsByLibraryItemId(item.id, ctx.uid) + if (item.labelNames && item.labelNames.length > 0) { + return findLabelsByLibraryItemId(item.id, ctx.uid) + } + + return [] }, async recommendations( item: { id: string recommendations?: Recommendation[] + recommenderNames?: string[] | null }, _: unknown, ctx: WithDataSourcesContext ) { if (item.recommendations) return item.recommendations - const recommendations = await findRecommendationsByLibraryItemId( - item.id, - ctx.uid - ) - return recommendations.map(recommandationDataToRecommendation) + if (item.recommenderNames && item.recommenderNames.length > 0) { + const recommendations = await findRecommendationsByLibraryItemId( + item.id, + ctx.uid + ) + return recommendations.map(recommandationDataToRecommendation) + } + + return [] }, }, Subscription: { diff --git a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts index 4feb518de..f9dccfa37 100644 --- a/packages/api/src/resolvers/importers/uploadImportFileResolver.ts +++ b/packages/api/src/resolvers/importers/uploadImportFileResolver.ts @@ -33,14 +33,14 @@ export const uploadImportFileResolver = authorized< UploadImportFileSuccess, UploadImportFileError, MutationUploadImportFileArgs ->(async (_, { type, contentType }, { claims: { uid }, log }) => { +>(async (_, { type, contentType }, { uid }) => { if (!VALID_CONTENT_TYPES.includes(contentType)) { return { errorCodes: [UploadImportFileErrorCode.BadRequest], } } - const user = await userRepository.findOneBy({ id: uid }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [UploadImportFileErrorCode.Unauthorized], diff --git a/packages/api/src/resolvers/popular_reads/index.ts b/packages/api/src/resolvers/popular_reads/index.ts index 3f91fe2c9..30e1de08c 100644 --- a/packages/api/src/resolvers/popular_reads/index.ts +++ b/packages/api/src/resolvers/popular_reads/index.ts @@ -11,12 +11,12 @@ export const addPopularReadResolver = authorized< AddPopularReadError, MutationAddPopularReadArgs >(async (_, { name }, { uid }) => { - const item = await addPopularRead(uid, name) - if (!item) { + const items = await addPopularRead(uid, name) + if (items.length === 0) { return { errorCodes: [AddPopularReadErrorCode.NotFound] } } return { - pageId: item.id, + pageId: items[0].id, } }) diff --git a/packages/api/src/resolvers/reaction/index.ts b/packages/api/src/resolvers/reaction/index.ts index aec16962a..ceea66324 100644 --- a/packages/api/src/resolvers/reaction/index.ts +++ b/packages/api/src/resolvers/reaction/index.ts @@ -17,7 +17,7 @@ export type PartialCreateReactionSuccess = Merge< // if ((!userArticleId && !highlightId) || (userArticleId && highlightId)) { // // One reaction target is required -// // Higlight replies hasn't supported yet +// // Highlight replies hasn't supported yet // return { // errorCodes: [CreateReactionErrorCode.BadTarget], // } diff --git a/packages/api/src/resolvers/recommendations/index.ts b/packages/api/src/resolvers/recommendations/index.ts index d3062dc5d..31f7da898 100644 --- a/packages/api/src/resolvers/recommendations/index.ts +++ b/packages/api/src/resolvers/recommendations/index.ts @@ -48,9 +48,7 @@ export const createGroupResolver = authorized< MutationCreateGroupArgs >(async (_, { input }, { uid, log }) => { try { - const userData = await userRepository.findOneBy({ - id: uid, - }) + const userData = await userRepository.findById(uid) if (!userData) { return { errorCodes: [CreateGroupErrorCode.Unauthorized], @@ -107,9 +105,7 @@ export const createGroupResolver = authorized< export const groupsResolver = authorized( async (_, __, { uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [GroupsErrorCode.Unauthorized], diff --git a/packages/api/src/resolvers/save/index.ts b/packages/api/src/resolvers/save/index.ts index 2acf0ce3b..f5508663d 100644 --- a/packages/api/src/resolvers/save/index.ts +++ b/packages/api/src/resolvers/save/index.ts @@ -18,9 +18,9 @@ export const savePageResolver = authorized< SaveSuccess, SaveError, MutationSavePageArgs ->(async (_, { input }, ctx) => { +>(async (_, { input }, { uid }) => { analytics.track({ - userId: ctx.uid, + userId: uid, event: 'link_saved', properties: { url: input.url, @@ -30,9 +30,7 @@ export const savePageResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: ctx.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } @@ -44,11 +42,7 @@ export const saveUrlResolver = authorized< SaveSuccess, SaveError, MutationSaveUrlArgs ->(async (_, { input }, ctx) => { - const { - claims: { uid }, - } = ctx - +>(async (_, { input }, { uid }) => { analytics.track({ userId: uid, event: 'link_saved', @@ -60,9 +54,7 @@ export const saveUrlResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } @@ -74,9 +66,9 @@ export const saveFileResolver = authorized< SaveSuccess, SaveError, MutationSaveFileArgs ->(async (_, { input }, ctx) => { +>(async (_, { input }, { uid }) => { analytics.track({ - userId: ctx.uid, + userId: uid, event: 'link_saved', properties: { url: input.url, @@ -86,9 +78,7 @@ export const saveFileResolver = authorized< }, }) - const user = await userRepository.findOneBy({ - id: ctx.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SaveErrorCode.Unauthorized] } } diff --git a/packages/api/src/resolvers/send_install_instructions/index.ts b/packages/api/src/resolvers/send_install_instructions/index.ts index 6435b807b..62d134fbc 100644 --- a/packages/api/src/resolvers/send_install_instructions/index.ts +++ b/packages/api/src/resolvers/send_install_instructions/index.ts @@ -14,11 +14,9 @@ const INSTALL_INSTRUCTIONS_EMAIL_TEMPLATE_ID = export const sendInstallInstructionsResolver = authorized< SendInstallInstructionsSuccess, SendInstallInstructionsError ->(async (_parent, _args, { claims, log }) => { +>(async (_parent, _args, { uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: claims.uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [SendInstallInstructionsErrorCode.Unauthorized] } diff --git a/packages/api/src/resolvers/subscriptions/index.ts b/packages/api/src/resolvers/subscriptions/index.ts index db249b9b9..8d5b74a83 100644 --- a/packages/api/src/resolvers/subscriptions/index.ts +++ b/packages/api/src/resolvers/subscriptions/index.ts @@ -89,7 +89,8 @@ export const subscriptionsResolver = authorized< } const subscriptions = await queryBuilder - .orderBy(`subscription.${sortBy}`, sortOrder, 'NULLS LAST') + .orderBy('subscription.status', 'ASC') + .addOrderBy(`subscription.${sortBy}`, sortOrder, 'NULLS LAST') .getMany() return { @@ -174,26 +175,8 @@ export const subscribeResolver = authorized< SubscribeSuccessPartial, SubscribeError, MutationSubscribeArgs ->(async (_, { input }, { authTrx, uid, log }) => { - log.info('subscribeResolver') - +>(async (_, { input }, { uid, log }) => { try { - // find existing subscription - const subscription = await authTrx((t) => - t.getRepository(Subscription).findOneBy({ - url: input.url || undefined, - name: input.name || undefined, - user: { id: uid }, - status: SubscriptionStatus.Active, - type: input.subscriptionType || SubscriptionType.Rss, // default to rss - }) - ) - if (subscription) { - return { - errorCodes: [SubscribeErrorCode.AlreadySubscribed], - } - } - analytics.track({ userId: uid, event: 'subscribed', @@ -203,49 +186,83 @@ export const subscribeResolver = authorized< }, }) - // create new rss subscription - if (input.url) { - const MAX_RSS_SUBSCRIPTIONS = 150 - // validate rss feed - const feed = await parser.parseURL(input.url) - - // limit number of rss subscriptions to 50 - const newSubscriptions = (await authTrx((t) => - t.query( - `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) - select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions - where user_id = $5 and type = 'RSS' and status = 'ACTIVE' - having count(*) < $7 - returning *;`, - [ - feed.title, - input.url, - feed.description || null, - SubscriptionType.Rss, - uid, - feed.image?.url || null, - MAX_RSS_SUBSCRIPTIONS, - ] - ) - )) as Subscription[] - - if (newSubscriptions.length === 0) { + // find existing subscription + const existingSubscription = await getRepository(Subscription).findOneBy({ + url: input.url, + user: { id: uid }, + type: SubscriptionType.Rss, + }) + if (existingSubscription) { + if (existingSubscription.status === SubscriptionStatus.Active) { return { - errorCodes: [SubscribeErrorCode.ExceededMaxSubscriptions], + errorCodes: [SubscribeErrorCode.AlreadySubscribed], } } - // create a cloud task to fetch rss feed item for the new subscription - await enqueueRssFeedFetch(uid, newSubscriptions[0]) + // re-subscribe + const updatedSubscription = await getRepository(Subscription).save({ + ...existingSubscription, + status: SubscriptionStatus.Active, + }) + + // create a cloud task to fetch rss feed item for resub subscription + await enqueueRssFeedFetch({ + userIds: [uid], + url: input.url, + subscriptionIds: [updatedSubscription.id], + scheduledDates: [new Date()], // fetch immediately + fetchedDates: [updatedSubscription.lastFetchedAt || null], + checksums: [updatedSubscription.lastFetchedChecksum || null], + }) return { - subscriptions: newSubscriptions, + subscriptions: [updatedSubscription], } } - log.info('missing url or name') + // create new rss subscription + const MAX_RSS_SUBSCRIPTIONS = 150 + // validate rss feed + const feed = await parser.parseURL(input.url) + + // limit number of rss subscriptions to 150 + const results = (await getRepository(Subscription).query( + `insert into omnivore.subscriptions (name, url, description, type, user_id, icon) + select $1, $2, $3, $4, $5, $6 from omnivore.subscriptions + where user_id = $5 and type = 'RSS' and status = 'ACTIVE' + having count(*) < $7 + returning *;`, + [ + feed.title, + input.url, + feed.description || null, + SubscriptionType.Rss, + uid, + feed.image?.url || null, + MAX_RSS_SUBSCRIPTIONS, + ] + )) as Subscription[] + + if (results.length === 0) { + return { + errorCodes: [SubscribeErrorCode.ExceededMaxSubscriptions], + } + } + + const newSubscription = results[0] + + // create a cloud task to fetch rss feed item for the new subscription + await enqueueRssFeedFetch({ + userIds: [uid], + url: input.url, + subscriptionIds: [newSubscription.id], + scheduledDates: [new Date()], // fetch immediately + fetchedDates: [null], + checksums: [null], + }) + return { - errorCodes: [SubscribeErrorCode.BadRequest], + subscriptions: [newSubscription], } } catch (error) { log.error('failed to subscribe', error) @@ -290,7 +307,11 @@ export const updateSubscriptionResolver = authorized< lastFetchedAt: input.lastFetchedAt ? new Date(input.lastFetchedAt) : undefined, + lastFetchedChecksum: input.lastFetchedChecksum || undefined, status: input.status || undefined, + scheduledAt: input.scheduledAt + ? new Date(input.scheduledAt) + : undefined, }) return repo.findOneByOrFail({ diff --git a/packages/api/src/resolvers/upload_files/index.ts b/packages/api/src/resolvers/upload_files/index.ts index 36169b5c4..9ea214584 100644 --- a/packages/api/src/resolvers/upload_files/index.ts +++ b/packages/api/src/resolvers/upload_files/index.ts @@ -24,7 +24,6 @@ import { contentReaderForLibraryItem, generateUploadFilePathName, generateUploadSignedUrl, - getFilePublicUrl, } from '../../utils/uploads' const isFileUrl = (url: string): boolean => { @@ -110,13 +109,12 @@ export const uploadFileRequestResolver = authorized< input.contentType ) - const publicUrl = getFilePublicUrl(uploadFilePathName) - - // If this is a file URL, we swap in the GCS public URL + // If this is a file URL, we swap in a special URL + const attachmentUrl = `https://omnivore.app/attachments/${uploadFilePathName}` if (isFileUrl(input.url)) { await authTrx(async (tx) => { await tx.getRepository(UploadFile).update(uploadFileId, { - url: publicUrl, + url: attachmentUrl, status: UploadFileStatus.Initialized, }) }) @@ -142,8 +140,8 @@ export const uploadFileRequestResolver = authorized< const uploadFileId = uploadFileData.id const item = await createLibraryItem( { - originalUrl: isFileUrl(input.url) ? publicUrl : input.url, id: input.clientRequestId || undefined, + originalUrl: isFileUrl(input.url) ? attachmentUrl : input.url, user: { id: uid }, title, readableContent: '', diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index 8b80e1340..38b0111e2 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -1,5 +1,9 @@ import * as jwt from 'jsonwebtoken' -import { RegistrationType, User as UserEntity } from '../../entity/user' +import { + RegistrationType, + StatusType, + User as UserEntity, +} from '../../entity/user' import { env } from '../../env' import { DeleteAccountError, @@ -38,6 +42,7 @@ import { import { userRepository } from '../../repository/user' import { createUser } from '../../services/create_user' import { sendVerificationEmail } from '../../services/send_emails' +import { updateUser } from '../../services/user' import { authorized, userDataToUser } from '../../utils/helpers' import { validateUsername } from '../../utils/usernamePolicy' import { WithDataSourcesContext } from '../types' @@ -47,9 +52,7 @@ export const updateUserResolver = authorized< UpdateUserError, MutationUpdateUserArgs >(async (_, { input: { name, bio } }, { uid, authTrx }) => { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { errorCodes: [UpdateUserErrorCode.UserNotFound] } } @@ -87,9 +90,7 @@ export const updateUserProfileResolver = authorized< UpdateUserProfileError, MutationUpdateUserProfileArgs >(async (_, { input: { userId, username, pictureUrl } }, { uid, authTrx }) => { - const user = await userRepository.findOneBy({ - id: userId, - }) + const user = await userRepository.findById(userId) if (!user) { return { errorCodes: [UpdateUserProfileErrorCode.Unauthorized] } } @@ -112,6 +113,7 @@ export const updateUserProfileResolver = authorized< profile: { username: lowerCasedUsername, }, + status: StatusType.Active, }) if (existingUser?.id) { return { @@ -156,6 +158,7 @@ export const googleLoginResolver: ResolverFn< const user = await userRepository.findOneBy({ email, + status: StatusType.Active, }) if (!user?.id) { return { errorCodes: [LoginErrorCode.UserNotFound] } @@ -251,9 +254,7 @@ export const getMeUserResolver: ResolverFn< return undefined } - const user = await userRepository.findOneBy({ - id: claims.uid, - }) + const user = await userRepository.findById(claims.uid) if (!user) { return undefined } @@ -277,12 +278,17 @@ export const getUserResolver: ResolverFn< const userId = id || (username && - (await userRepository.findOneBy({ profile: { username } }))?.id) + ( + await userRepository.findOneBy({ + profile: { username }, + status: StatusType.Active, + }) + )?.id) if (!userId) { return { errorCodes: [UserErrorCode.UserNotFound] } } - const userRecord = await userRepository.findOneBy({ id: userId }) + const userRecord = await userRepository.findById(userId) if (!userRecord) { return { errorCodes: [UserErrorCode.UserNotFound] } } @@ -313,9 +319,10 @@ export const deleteAccountResolver = authorized< DeleteAccountSuccess, DeleteAccountError, MutationDeleteAccountArgs ->(async (_, { userID }, { authTrx, log }) => { - const result = await authTrx(async (t) => { - return t.withRepository(userRepository).delete(userID) +>(async (_, { userID }, { log }) => { + // soft delete user + const result = await updateUser(userID, { + status: StatusType.Deleted, }) if (!result.affected) { log.error('Error deleting user account') @@ -334,9 +341,7 @@ export const updateEmailResolver = authorized< MutationUpdateEmailArgs >(async (_, { input: { email } }, { authTrx, uid, log }) => { try { - const user = await userRepository.findOneBy({ - id: uid, - }) + const user = await userRepository.findById(uid) if (!user) { return { diff --git a/packages/api/src/routers/auth/apple_auth.ts b/packages/api/src/routers/auth/apple_auth.ts index 925e7d37e..99590ff4d 100644 --- a/packages/api/src/routers/auth/apple_auth.ts +++ b/packages/api/src/routers/auth/apple_auth.ts @@ -15,6 +15,7 @@ import { suggestedUsername, } from './jwt_helpers' import { analytics } from '../../utils/analytics' +import { StatusType } from '../../entity/user' const appleBaseURL = 'https://appleid.apple.com' const audienceName = 'app.omnivore.app' @@ -122,6 +123,7 @@ export async function handleAppleWebAuth( const user = await userRepository.findOneBy({ sourceUserId: decodedTokenResult.sourceUserId, source: 'APPLE', + status: StatusType.Active, }) const userId = user?.id diff --git a/packages/api/src/routers/auth/auth_router.ts b/packages/api/src/routers/auth/auth_router.ts index 366c10ada..7779b9af8 100644 --- a/packages/api/src/routers/auth/auth_router.ts +++ b/packages/api/src/routers/auth/auth_router.ts @@ -46,6 +46,7 @@ import { } from './google_auth' import { createWebAuthToken } from './jwt_helpers' import { createMobileAccountCreationResponse } from './mobile/account_creation' +import rateLimit from 'express-rate-limit' export interface SignupRequest { email: string @@ -80,6 +81,15 @@ export const isValidSignupRequest = (obj: any): obj is SignupRequest => { ) } +// The hourly limiter is used on the create account, +// and reset password endpoints +// this limits users to five operations per an hour +const hourlyLimiter = rateLimit({ + windowMs: 60 * 60 * 1000, + max: 5, + skip: (req) => env.dev.isLocal, +}) + export function authRouter() { const router = express.Router() @@ -108,6 +118,7 @@ export function authRouter() { ) router.post( '/create-account', + hourlyLimiter, cors(corsConfig), async (req, res) => { const { name, bio, username } = req.body @@ -421,7 +432,7 @@ export function authRouter() { const { email, password } = req.body try { const user = await userRepository.findByEmail(email.trim()) - if (!user?.id) { + if (!user || user.status === StatusType.Deleted) { return res.redirect( `${env.client.url}/auth/email-login?errorCodes=${LoginErrorCode.UserNotFound}` ) @@ -480,6 +491,7 @@ export function authRouter() { router.post( '/email-signup', + hourlyLimiter, cors(corsConfig), async (req: express.Request, res: express.Response) => { if (!isValidSignupRequest(req.body)) { @@ -599,6 +611,7 @@ export function authRouter() { router.post( '/forgot-password', + hourlyLimiter, cors(corsConfig), async (req: express.Request, res: express.Response) => { const email = req.body.email?.trim() as string // trim whitespace @@ -610,7 +623,7 @@ export function authRouter() { try { const user = await userRepository.findByEmail(email) - if (!user) { + if (!user || user.status === StatusType.Deleted) { return res.redirect(`${env.client.url}/auth/reset-sent`) } diff --git a/packages/api/src/routers/auth/google_auth.ts b/packages/api/src/routers/auth/google_auth.ts index 781c45dcd..1108b6f23 100644 --- a/packages/api/src/routers/auth/google_auth.ts +++ b/packages/api/src/routers/auth/google_auth.ts @@ -1,6 +1,7 @@ import { google, oauth2_v2 as oauthV2 } from 'googleapis' import { OAuth2Client } from 'googleapis-common' import url from 'url' +import { StatusType } from '../../entity/user' import { env, homePageURL } from '../../env' import { LoginErrorCode } from '../../generated/graphql' import { userRepository } from '../../repository/user' @@ -130,6 +131,7 @@ export async function handleGoogleWebAuth( const user = await userRepository.findOneBy({ email, source: 'GOOGLE', + status: StatusType.Active, }) const userId = user?.id diff --git a/packages/api/src/routers/auth/mobile/sign_in.ts b/packages/api/src/routers/auth/mobile/sign_in.ts index 40d4462bb..4af80f0a6 100644 --- a/packages/api/src/routers/auth/mobile/sign_in.ts +++ b/packages/api/src/routers/auth/mobile/sign_in.ts @@ -46,7 +46,7 @@ export async function createMobileEmailSignInResponse( } const user = await userRepository.findByEmail(email.trim()) - if (!user?.id || !user?.password) { + if (!user || !user.password || user.status === StatusType.Deleted) { throw new Error('user not found') } diff --git a/packages/api/src/routers/svc/content.ts b/packages/api/src/routers/svc/content.ts index ce184ea5b..70723429b 100644 --- a/packages/api/src/routers/svc/content.ts +++ b/packages/api/src/routers/svc/content.ts @@ -91,7 +91,7 @@ export function contentServiceRouter() { if (msg.description) itemToUpdate.description = msg.description // This event is fired after the file is fully uploaded, - // so along with updateing content, we mark it as + // so along with updating content, we mark it as // succeeded. itemToUpdate.state = LibraryItemState.Succeeded diff --git a/packages/api/src/routers/svc/email_attachment.ts b/packages/api/src/routers/svc/email_attachment.ts index b9bf3c77c..6c127135b 100644 --- a/packages/api/src/routers/svc/email_attachment.ts +++ b/packages/api/src/routers/svc/email_attachment.ts @@ -24,7 +24,6 @@ import { generateUploadFilePathName, generateUploadSignedUrl, getStorageFileDetails, - makeStorageFilePublic, } from '../../utils/uploads' export function emailAttachmentRouter() { @@ -143,11 +142,12 @@ export function emailAttachmentRouter() { return res.status(400).send('BAD REQUEST') } - const uploadFileUrlOverride = await makeStorageFilePublic( - uploadFileData.id, - uploadFileData.fileName + const uploadFilePathName = generateUploadFilePathName( + uploadFileId, + uploadFile.fileName ) + const uploadFileUrlOverride = `https://omnivore.app/attachments/${uploadFilePathName}` const uploadFileHash = uploadFileDetails.md5Hash const itemType = uploadFile.contentType === 'application/pdf' diff --git a/packages/api/src/routers/svc/rss_feed.ts b/packages/api/src/routers/svc/rss_feed.ts index 0834e5025..3b6a21247 100644 --- a/packages/api/src/routers/svc/rss_feed.ts +++ b/packages/api/src/routers/svc/rss_feed.ts @@ -4,7 +4,10 @@ import { Subscription } from '../../entity/subscription' import { SubscriptionStatus, SubscriptionType } from '../../generated/graphql' import { readPushSubscription } from '../../pubsub' import { getRepository } from '../../repository' -import { enqueueRssFeedFetch } from '../../utils/createTask' +import { + enqueueRssFeedFetch, + RssSubscriptionGroup, +} from '../../utils/createTask' import { logger } from '../../utils/logger' export function rssFeedRouter() { @@ -22,21 +25,33 @@ export function rssFeedRouter() { return res.status(200).send('Expired') } - // get all active rss feed subscriptions - const subscriptions = await getRepository(Subscription).find({ - select: ['id', 'url', 'user', 'lastFetchedAt'], - where: { - type: SubscriptionType.Rss, - status: SubscriptionStatus.Active, - }, - relations: ['user'], - }) + // get active rss feed subscriptions scheduled for fetch and group by feed url + const subscriptionGroups = (await getRepository(Subscription).query( + ` + SELECT + url, + ARRAY_AGG(id) AS "subscriptionIds", + ARRAY_AGG(user_id) AS "userIds", + ARRAY_AGG(last_fetched_at) AS "fetchedDates", + ARRAY_AGG(coalesce(scheduled_at, NOW())) AS "scheduledDates", + ARRAY_AGG(last_fetched_checksum) AS checksums + FROM + omnivore.subscriptions + WHERE + type = $1 + AND status = $2 + AND (scheduled_at <= NOW() OR scheduled_at IS NULL) + GROUP BY + url + `, + [SubscriptionType.Rss, SubscriptionStatus.Active] + )) as RssSubscriptionGroup[] // create a cloud taks to fetch rss feed item for each subscription await Promise.all( - subscriptions.map((subscription) => { + subscriptionGroups.map((subscriptionGroup) => { try { - return enqueueRssFeedFetch(subscription.user.id, subscription) + return enqueueRssFeedFetch(subscriptionGroup) } catch (error) { logger.info('error creating rss feed fetch task', error) } diff --git a/packages/api/src/routers/user_router.ts b/packages/api/src/routers/user_router.ts index cc830fc71..8e13ee184 100644 --- a/packages/api/src/routers/user_router.ts +++ b/packages/api/src/routers/user_router.ts @@ -39,7 +39,7 @@ export function userRouter() { return } try { - const user = await userRepository.findOneBy({ id: claims.uid }) + const user = await userRepository.findById(claims.uid) if (!user) { res.status(400).send('Bad Request') return diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e3b2b3442..ff9d75664 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -637,6 +637,7 @@ const schema = gql` readingProgressTopPercent: Float readingProgressPercent: Float! readingProgressAnchorIndex: Int + force: Boolean } enum SaveArticleReadingProgressErrorCode { NOT_FOUND @@ -2538,8 +2539,7 @@ const schema = gql` } input SubscribeInput { - url: String - name: String + url: String! subscriptionType: SubscriptionType } @@ -2548,7 +2548,9 @@ const schema = gql` name: String description: String lastFetchedAt: Date + lastFetchedChecksum: String status: SubscriptionStatus + scheduledAt: Date } union UpdateSubscriptionResult = diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index cc757acc3..b40a82bb8 100755 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -58,6 +58,9 @@ export const createApp = (): { app.use(json({ limit: '100mb' })) app.use(urlencoded({ limit: '100mb', extended: true })) + // set to true if behind a reverse proxy/load balancer + app.set('trust proxy', env.server.trustProxy) + const apiLimiter = rateLimit({ windowMs: 60 * 1000, // 1 minute max: async (req) => { @@ -65,7 +68,7 @@ export const createApp = (): { const token = getTokenByRequest(req) try { const claims = await getClaimsByToken(token) - return claims ? 100 : 15 + return claims ? 60 : 15 } catch (e) { console.log('non-authenticated request') return 15 diff --git a/packages/api/src/services/groups.ts b/packages/api/src/services/groups.ts index 0a3de5891..e3ec84ebe 100644 --- a/packages/api/src/services/groups.ts +++ b/packages/api/src/services/groups.ts @@ -280,3 +280,7 @@ export const getGroupsWhereUserCanPost = async ( .innerJoinAndSelect('members.user', 'user') .getMany() } + +export const deleteGroup = async (groupId: string) => { + return getRepository(Group).delete(groupId) +} diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 07be9f885..7fec735e0 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -431,13 +431,88 @@ export const updateLibraryItem = async ( await pubsub.entityUpdated>( EntityType.PAGE, - { ...libraryItem, id }, + { + ...libraryItem, + id, + // don't send original content and readable content + originalContent: undefined, + readableContent: undefined, + }, userId ) return updatedLibraryItem } +export const updateLibraryItemReadingProgress = async ( + id: string, + userId: string, + bottomPercent: number, + topPercent: number | null = null, + anchorIndex: number | null = null, + pubsub = createPubSubClient() +): Promise => { + // If we have a top percent, we only save it if it's greater than the current top percent + // or set to zero if the top percent is zero. + const result = (await authTrx( + async (tx) => + tx.getRepository(LibraryItem).query( + ` + UPDATE omnivore.library_item + SET reading_progress_top_percent = CASE + WHEN reading_progress_top_percent < $2 THEN $2 + WHEN $2 = 0 THEN 0 + ELSE reading_progress_top_percent + END, + reading_progress_bottom_percent = CASE + WHEN reading_progress_bottom_percent < $3 THEN $3 + WHEN $3 = 0 THEN 0 + ELSE reading_progress_bottom_percent + END, + reading_progress_highest_read_anchor = CASE + WHEN reading_progress_top_percent < $4 THEN $4 + WHEN $4 = 0 THEN 0 + ELSE reading_progress_highest_read_anchor + END, + read_at = now() + WHERE id = $1 AND ( + (reading_progress_top_percent < $2 OR $2 = 0) OR + (reading_progress_bottom_percent < $3 OR $3 = 0) OR + (reading_progress_highest_read_anchor < $4 OR $4 = 0) + ) + RETURNING + id, + reading_progress_top_percent as "readingProgressTopPercent", + reading_progress_bottom_percent as "readingProgressBottomPercent", + reading_progress_highest_read_anchor as "readingProgressHighestReadAnchor", + read_at as "readAt" + `, + [id, topPercent, bottomPercent, anchorIndex] + ), + undefined, + userId + )) as [LibraryItem[], number] + if (result[1] === 0) { + return null + } + + const updatedItem = result[0][0] + await pubsub.entityUpdated>( + EntityType.PAGE, + { + id, + readingProgressBottomPercent: updatedItem.readingProgressBottomPercent, + readingProgressTopPercent: updatedItem.readingProgressTopPercent, + readingProgressHighestReadAnchor: + updatedItem.readingProgressHighestReadAnchor, + readAt: updatedItem.readAt, + }, + userId + ) + + return updatedItem +} + export const createLibraryItems = async ( libraryItems: DeepPartial[], userId: string @@ -452,7 +527,8 @@ export const createLibraryItems = async ( export const createLibraryItem = async ( libraryItem: DeepPartial, userId: string, - pubsub = createPubSubClient() + pubsub = createPubSubClient(), + skipPubSub = false ): Promise => { const newLibraryItem = await authTrx( async (tx) => @@ -466,9 +542,18 @@ export const createLibraryItem = async ( userId ) - await pubsub.entityCreated( + if (skipPubSub) { + return newLibraryItem + } + + await pubsub.entityCreated>( EntityType.PAGE, - newLibraryItem, + { + ...newLibraryItem, + // don't send original content and readable content + originalContent: undefined, + readableContent: undefined, + }, userId ) diff --git a/packages/api/src/services/newsletters.ts b/packages/api/src/services/newsletters.ts index fb9e33cca..f492cf4b5 100644 --- a/packages/api/src/services/newsletters.ts +++ b/packages/api/src/services/newsletters.ts @@ -21,10 +21,7 @@ export const createNewsletterEmail = async ( userId: string, confirmationCode?: string ): Promise => { - const user = await userRepository.findOne({ - where: { id: userId }, - relations: ['profile'], - }) + const user = await userRepository.findById(userId) if (!user) { return Promise.reject({ errorCode: CreateNewsletterEmailErrorCode.Unauthorized, diff --git a/packages/api/src/services/popular_reads.ts b/packages/api/src/services/popular_reads.ts index c98fd9851..de6daab4d 100644 --- a/packages/api/src/services/popular_reads.ts +++ b/packages/api/src/services/popular_reads.ts @@ -1,93 +1,23 @@ import * as httpContext from 'express-http-context2' -import { readFileSync } from 'fs' -import path from 'path' -import { DeepPartial, EntityManager } from 'typeorm' +import { EntityManager } from 'typeorm' import { appDataSource } from '../data_source' -import { LibraryItem } from '../entity/library_item' -import { PageType } from '../generated/graphql' import { authTrx } from '../repository' import { libraryItemRepository } from '../repository/library_item' -import { generateSlug, stringToHash, wordsCount } from '../utils/helpers' import { logger } from '../utils/logger' -import { createLibraryItem } from './library_item' -type PopularRead = { - url: string - title: string - author: string - description: string - previewImage?: string - publishedAt: Date - siteName: string - - content: string - originalHtml: string -} - -const popularRead = (key: string): PopularRead | undefined => { - const metadata = popularReads.find((pr) => pr.key === key) - if (!metadata) { - return undefined - } - - try { - const content = readFileSync( - path.resolve(__dirname, `popular_reads/${key}-content.html`), - 'utf8' - ) - const originalHtml = readFileSync( - path.resolve(__dirname, `./popular_reads/${key}-original.html`), - 'utf8' - ) - if (!content || !originalHtml) { - return undefined - } - - return { - ...metadata, - content, - originalHtml, - } - } catch (e) { - logger.info('error adding popular read', e) - return undefined - } -} - -const popularReadToLibraryItem = ( +export const addPopularRead = async ( + userId: string, name: string, - userId: string -): DeepPartial | null => { - const pr = popularRead(name) - if (!pr) { - return null - } - - return { - slug: generateSlug(pr.title), - readableContent: pr.content, - originalContent: pr.originalHtml, - description: pr.description, - title: pr.title, - author: pr.author, - originalUrl: pr.url, - itemType: PageType.Article, - textContentHash: stringToHash(pr.content), - thumbnail: pr.previewImage, - publishedAt: pr.publishedAt, - siteName: pr.siteName, - user: { id: userId }, - wordCount: wordsCount(pr.content), - } -} - -export const addPopularRead = async (userId: string, name: string) => { - const itemToSave = popularReadToLibraryItem(name, userId) - if (!itemToSave) { - return null - } - - return createLibraryItem(itemToSave, userId) + entityManager?: EntityManager +) => { + return authTrx( + async (tx) => + tx + .withRepository(libraryItemRepository) + .createByPopularRead(name, userId), + entityManager, + userId + ) } const addPopularReads = async ( @@ -95,15 +25,15 @@ const addPopularReads = async ( userId: string, entityManager: EntityManager ) => { - const libraryItems = names - .map((name) => popularReadToLibraryItem(name, userId)) - .filter((pr) => pr !== null) as DeepPartial[] - - return authTrx( - async (tx) => tx.withRepository(libraryItemRepository).save(libraryItems), - entityManager, - userId - ) + // insert one by one to ensure that the order is preserved + for (const name of names) { + try { + await addPopularRead(userId, name, entityManager) + } catch (error) { + logger.error('failed to add popular read', error) + continue + } + } } export const addPopularReadsForNewUser = async ( @@ -132,81 +62,3 @@ export const addPopularReadsForNewUser = async ( defaultReads.push('omnivore_get_started') await addPopularReads(defaultReads, userId, em) } - -const popularReads = [ - { - key: 'omnivore_get_started', - url: 'https://blog.omnivore.app/p/getting-started-with-omnivore', - title: 'Getting Started with Omnivore', - author: 'The Omnivore Team', - description: 'Get the most out of Omnivore by learning how to use it.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sxQnqya1QNApB7ZAGPj9K20AU6sw0UAnjmAIy2ub8hUU/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F658efff4-341a-4720-8cf6-9b2bdbedfaa7_800x668.gif', - publishedAt: new Date('2021-10-13'), - siteName: 'Omnivore Blog', - }, - { - key: 'omnivore_ios', - url: 'https://blog.omnivore.app/p/saving-links-from-your-iphone-or', - title: 'Saving Links from Your iPhone or iPad', - author: 'Omnivore', - description: 'Learn how to save articles on iOS.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sWDfv7sARTIdAlx6Rw_6t-QwL3T9aniEJRa1-jVaglNg/https://substackcdn.com/image/youtube/w_728,c_limit/k6RkIqepAig', - publishedAt: new Date('2021-10-19'), - siteName: 'Omnivore Blog', - }, - { - key: 'omnivore_organize', - url: 'https://blog.omnivore.app/p/organize-your-omnivore-library-with', - title: 'Organize your Omnivore library with labels', - author: 'The Omnivore Team', - description: 'Use labels to organize your Omnivore library.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sTgJ5Q0XIg_EHdmPWcxtXFmkjn8T6hkJt7S9ziClagYo/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fdaf07af7-5cdb-4ecc-aace-1a46de3e9c58_1827x1090.png', - publishedAt: new Date('2022-04-18'), - siteName: 'Omnivore Blog', - }, - { - key: 'rlove_carnitas', - url: 'https://medium.com/@rlove/carnitas-ff0ef1044ae9', - title: 'Slow-Braised Carnitas Recipe', - author: 'Robert Love', - description: - 'Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. This is not an authentic recipe.', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/88x88,sIcDXt3Ar0baKG1e1Yi1e2VUZFL85xPlOeEfAxF-s-Nw/https://miro.medium.com/max/1200/1*Wl-dMBJpSgPUxUOnPQthyg.jpeg', - publishedAt: new Date('2017-02-24'), - siteName: '@rlove', - }, - { - key: 'power_read_it_later', - url: 'https://fortelabs.co/blog/the-secret-power-of-read-it-later-apps', - title: 'The Secret Power of ‘Read It Later’ Apps', - author: 'Tiago Forte', - description: - 'At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket', - previewImage: - 'https://proxy-prod.omnivore-image-cache.app/320x320,sGN5R34M5z068QMXDZD32CQD6mCbxc47hWXm__JVUePE/https://fortelabs.com/wp-content/uploads/2015/11/1rPXwIczUJRCE54v8FfAHGw.jpeg', - publishedAt: new Date('2022-01-24'), - siteName: 'Forte Labs', - }, - { - key: 'elad_meetings', - url: 'http://blog.eladgil.com/2018/07/meeting-etiquette.html', - title: 'Better Meetings', - author: 'Elad Gil', - description: 'How to make meetings more productive.', - publishedAt: new Date('2018-07-02'), - siteName: 'Elad Blog', - }, - { - key: 'jonbo_digital_tools', - url: 'https://jon.bo/posts/digital-tools/', - title: 'Digital Tools I Wish Existed', - author: 'Jonathan Borichevskiy', - description: `My digital life in a nutshell: I discover relevant content I don’t have time to consume...`, - publishedAt: new Date('2019-11-28'), - siteName: 'JON.BO', - }, -] diff --git a/packages/api/src/services/popular_reads/elad_meetings-content.html b/packages/api/src/services/popular_reads/elad_meetings-content.html deleted file mode 100644 index 34f8a7aee..000000000 --- a/packages/api/src/services/popular_reads/elad_meetings-content.html +++ /dev/null @@ -1,36 +0,0 @@ -
-
-

As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
-
1. Determine who is necessary in the meeting.
- Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
-
2. Send out an agenda in advance.
- What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
-
Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
-
3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
- If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
-
If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
-
4. Kick off the meeting with objectives.
- Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
-
5. Assign a note taker.
- Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
-
6. Send out meeting notes.
- Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
-
Meeting notes would optimally include:
- a. Subject/topic of meeting
- b. Date
- c. Attendees
- d. Actions/decisions
- e. Agenda
- f. Detailed notes
-
7. Clean up the meeting calendar ongoing.
- Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
-
NOTES
- [1] There may be legal reasons to not take notes in specific meetings. Consult with the company's GC or external counsel on training for meeting note etiquette. -

-

MY BOOK -

-

You can pre-order the High Growth Handbook here.
-
-

-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/elad_meetings-original.html b/packages/api/src/services/popular_reads/elad_meetings-original.html deleted file mode 100644 index 054b6f5f9..000000000 --- a/packages/api/src/services/popular_reads/elad_meetings-original.html +++ /dev/null @@ -1,1497 +0,0 @@ - - - - - - - - - - - - - - - - - - Elad Blog: Better Meetings - - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

- Monday, July 2, 2018 -

-
-
-
- - -

- Better Meetings -

-
-
-
-
-
- As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
-
1. Determine who is necessary in the meeting.
- Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
-
2. Send out an agenda in advance.
- What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
-
Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
-
3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
- If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
-
If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
-
4. Kick off the meeting with objectives.
- Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
-
5. Assign a note taker.
- Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
-
6. Send out meeting notes.
- Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
-
Meeting notes would optimally include:
- a. Subject/topic of meeting
- b. Date
- c. Attendees
- d. Actions/decisions
- e. Agenda
- f. Detailed notes
-
7. Clean up the meeting calendar ongoing.
- Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
-
NOTES
- [1] There may be legal reasons to not take notes in specific meetings. Consult with the company's GC or external counsel on training for meeting note etiquette.
-
-
-
-
- MY BOOK -
You can pre-order the High Growth Handbook here.
-

-
-
-
-
- RELATED POSTS -
- - - - - - -
-
-
-
-
- Posted by at - -
- -
-
-
-
-
-
-
- -
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- - - - - - - - -
- - - -
- -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- - - - - - diff --git a/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html b/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html deleted file mode 100644 index 711fe33ef..000000000 --- a/packages/api/src/services/popular_reads/jonbo_digital_tools-content.html +++ /dev/null @@ -1,208 +0,0 @@ -
-
-

My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later.

-

There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed:

-
    -
  • -

    The more seamless the acquisition & ingestion, the more engaged I am with the content

    -
    • -
  • -

    Insights are just as likely to be found in a 400-page book as in a 40-minute podcast

    -
    • -
  • -

    Notes and their subsequent review are essential for long-term retention

    -
    • -
  • -

    Recommendations from other humans are as good, if not better, than algorithmic suggestions

    -
    • -
-

In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here.

-

Queue management for inbound digital content # -

-

Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance:

-
    -
  • -

    Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more.

    -
    • -
  • -

    Every book, article, post, or tweet has the potential to lead to more content.

    -
    • -
  • -

    Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs.

    -
    • -
  • -

    I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee?

    -
    • -
  • -

    Learning, work, news, and entertainment all have different priorities in my life (roughly in that order).

    -
    • -
  • -

    I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month.

    -
    • -
  • -

    I’m not always connected to a stable internet connection.

    -
    • -
  • -

    If it’s a long piece of content I want my position saved reliably so I can resume at a later point.

    -
    • -
  • -

    I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation.

    -
    • -
  • -

    I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place.

    -
    • -
  • -

    I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat.

    -
    • -
-
- -
Relevant XKCD, as is tradition
-
-

Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one.

-

Honorable Mentions: Pocket, Instapaper -

-

A universal book log, recommendation & sharing system # -

-

I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof.

-

Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry.

-

Whatever this tool for managing my ever-growing reading list will be, it should:

-
    -
  • -

    Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests.

    -
    • -
  • -

    Allow me to tag books instead of placing them into static lists (think clusters or tag clouds).

    -
    • -
  • -

    Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to.

    -
    • -
  • -

    Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it.

    -
    • -
  • -

    Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was.

    -
    • -
  • -

    Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on.

    -
    • -
  • -

    Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from.

    -
    • -
-

Honorable Mentions: None :(

-

Intelligent PDF viewers, eBook readers, audiobook & podcast players # -

-
- -
Functionality I want in my document reader
-
-

Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to:

-
    -
  • -

    Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them.

    -
    • -
  • -

    See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition.

    -
    • -
  • -

    View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know!

    -
    • -
  • -

    Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019.

    -
    • -
  • -

    Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues.

    -
    • -
  • -

    Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points).

    -
    • -
  • -

    Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading.

    -
    • -
-
- -
What I want my audiobook player to look like
-
-

Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could:

-
    -
  • -

    Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street.

    -
    • -
  • -

    View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps?

    -
    • -
-

Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar -

-

A centralized search interface for my digital brain (memex) # -

-

I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything.

-

I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should:

-
    -
  • -

    Accept and parse the following queries:

    -
      -
    • -

      spacex announcement type:video 2016

      -
      • -
    • -

      links from:jon@test.org topic:python

      -
      • -
    • -

      paper on temperature, productivity referenced in book:Uninhabitable Earth

      -
      • -
    • -

      type:pdf habits digital interfaces

      -
      • -
    • -

      reading comprehension type:blog post

      -
      • -
    • -

      printer ink receipt

      -
      • -
    • -

      type:book read:2017 finance

      -
      • -
    • -

      file:py datetime parse

      -
      • -
    -
    • -
  • -

    Respect my privacy: hosted on something I control and never mined for ads.

    -
    • -
  • -

    Support all my devices with two-way sync so I can search and add to it wherever I am.

    -
    • -
  • -

    Be extensible: allow me to easily ingest my own information and extend with desired functionality.

    -
    • -
  • -

    Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of.

    -
    • -
  • -

    Notify me about changes to documents and webpages I’ve visited.

    -
    • -
  • -

    Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it.

    -
    • -
-

Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop -

-

Parting Thoughts # -

-

I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools.

-

It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share.

-

I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback.

-

- HN Discussion -

-

Appendix # -

-

- The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS -

-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html b/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html deleted file mode 100644 index 66ff99044..000000000 --- a/packages/api/src/services/popular_reads/jonbo_digital_tools-original.html +++ /dev/null @@ -1,453 +0,0 @@ - - - - - Digital Tools I Wish Existed :: up & to the right — Jonathan Borichevskiy - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - -
-
-
-

- Digital Tools I Wish Existed -

-

-
- Published: 2019-11-28 -
#memex  #information  #learning  #web  #internet  -
-

- My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later. -

-

- There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed: -

-
    -
  • -

    - The more seamless the acquisition & ingestion, the more engaged I am with the content -

    -
    • -
  • -

    - Insights are just as likely to be found in a 400-page book as in a 40-minute podcast -

    -
    • -
  • -

    - Notes and their subsequent review are essential for long-term retention -

    -
    • -
  • -

    - Recommendations from other humans are as good, if not better, than algorithmic suggestions -

    -
    • -
-

- In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here. -

-

- Queue management for inbound digital content # -

-

- Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance: -

-
    -
  • -

    - Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more. -

    -
    • -
  • -

    - Every book, article, post, or tweet has the potential to lead to more content. -

    -
    • -
  • -

    - Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs. -

    -
    • -
  • -

    - I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee? -

    -
    • -
  • -

    - Learning, work, news, and entertainment all have different priorities in my life (roughly in that order). -

    -
    • -
  • -

    - I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month. -

    -
    • -
  • -

    - I’m not always connected to a stable internet connection. -

    -
    • -
  • -

    - If it’s a long piece of content I want my position saved reliably so I can resume at a later point. -

    -
    • -
  • -

    - I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation. -

    -
    • -
  • -

    - I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place. -

    -
    • -
  • -

    - I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat. -

    -
    • -
-
- -
- Relevant XKCD, as is tradition -
-
-

- Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one. -

-

- Honorable Mentions: Pocket, Instapaper -

-

- A universal book log, recommendation & sharing system # -

-

- I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof. -

-

- Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry. -

-

- Whatever this tool for managing my ever-growing reading list will be, it should: -

-
    -
  • -

    - Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests. -

    -
    • -
  • -

    - Allow me to tag books instead of placing them into static lists (think clusters or tag clouds). -

    -
    • -
  • -

    - Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to. -

    -
    • -
  • -

    - Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it. -

    -
    • -
  • -

    - Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was. -

    -
    • -
  • -

    - Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on. -

    -
    • -
  • -

    - Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from. -

    -
    • -
-

- Honorable Mentions: None :( -

-

- Intelligent PDF viewers, eBook readers, audiobook & podcast players # -

-
- -
- Functionality I want in my document reader -
-
-

- Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to: -

-
    -
  • -

    - Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them. -

    -
    • -
  • -

    - See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition. -

    -
    • -
  • -

    - View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know! -

    -
    • -
  • -

    - Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019. -

    -
    • -
  • -

    - Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues. -

    -
    • -
  • -

    - Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points). -

    -
    • -
  • -

    - Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading. -

    -
    • -
-
- -
- What I want my audiobook player to look like -
-
-

- Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could: -

-
    -
  • -

    - Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street. -

    -
    • -
  • -

    - View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps? -

    -
    • -
-

- Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar -

-

- A centralized search interface for my digital brain (memex) # -

-

- I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything. -

-

- I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should: -

-
    -
  • -

    - Accept and parse the following queries: -

    -
      -
    • -

      - spacex announcement type:video 2016 -

      -
      • -
    • -

      - links from:jon@test.org topic:python -

      -
      • -
    • -

      - paper on temperature, productivity referenced in book:Uninhabitable Earth -

      -
      • -
    • -

      - type:pdf habits digital interfaces -

      -
      • -
    • -

      - reading comprehension type:blog post -

      -
      • -
    • -

      - printer ink receipt -

      -
      • -
    • -

      - type:book read:2017 finance -

      -
      • -
    • -

      - file:py datetime parse -

      -
      • -
    -
    • -
  • -

    - Respect my privacy: hosted on something I control and never mined for ads. -

    -
    • -
  • -

    - Support all my devices with two-way sync so I can search and add to it wherever I am. -

    -
    • -
  • -

    - Be extensible: allow me to easily ingest my own information and extend with desired functionality. -

    -
    • -
  • -

    - Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of. -

    -
    • -
  • -

    - Notify me about changes to documents and webpages I’ve visited. -

    -
    • -
  • -

    - Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it. -

    -
    • -
-

- Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop -

-

- Parting Thoughts # -

-

- I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools. -

-

- It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share. -

-

- I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback. -

-

- HN Discussion -

-

- 2019-12-09: fixes grammar -

-

- Appendix # -

-

- The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS -

-
- -
-
- - - -
- - diff --git a/packages/api/src/services/popular_reads/omnivore_get_started-content.html b/packages/api/src/services/popular_reads/omnivore_get_started-content.html deleted file mode 100644 index d25355959..000000000 --- a/packages/api/src/services/popular_reads/omnivore_get_started-content.html +++ /dev/null @@ -1,331 +0,0 @@ -
-
-
-

Omnivore is a read-it-later app that lets you save and organize everything you read online.

-
-
-

This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities:

-
    -
  • -

    Saving

    -
    • -
  • -

    Reading

    -
    • -
  • -

    Organizing

    -
    • -
  • -

    Integrations

    -
    • -
-

- The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them. -

-

There are five ways to save links to pages or articles that you wish to read later:

-
    -
  • -

    Saving from Your Omnivore Library

    -
    • -
  • -

    Saving from a Browser 

    -
    • -
  • -

    Saving from a Phone or Tablet (iOS or Android)

    -
    • -
  • -

    Newsletter Subscriptions via Email

    -
    • -
  • -

    - Saving PDFs from a Mac
    -

    -
    • -
-

Saving from Your Omnivore Library

-

- 1. In the upper right corner of your Library, tap the Add Link button.
- 2. Enter the URL you wish to save and tap Add Link.
- 3. The link will appear in your Library the next time you refresh it.
-

-

Saving from a Browser

-

1. Download and install the Omnivore extension for your browser:

- -

- 2. Navigate to the page you wish to save and tap the Omnivore button in your browser’s toolbar or Extensions menu.
- 3. Alternatively, you can right-click (command+click on Mac) on any hyperlink and select Save to Omnivore from the menu.
- 4. The link will appear in your Library the next time you refresh it.
-

-

Saving from a Phone or Tablet

-

The best way to save links from your mobile device is via the Omnivore app. You can download the app here:

- -

Once the mobile app is installed:

-
    -
  1. -

    - In your browser, navigate to the page you wish to save and tap the Share button. -

    -
    2. -
  2. -

    - Tap the Omnivore icon in the Share menu. -

    -
    3. -
  3. -

    The link will appear in your Library the next time you refresh it.

    -
    4. -
-

Newsletter Subscriptions via Email

-

- 1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu. -

-

- 2. Tap Create a New Email Address to add a new email address (ex: username-123abc@inbox.omnivore.app) to the list. -

-

3. Click the Copy icon next to the email address.

-

- 4. Navigate to the signup page for the newsletter you wish to subscribe to.
- 5. Paste the Omnivore email address into the signup form. -

-

6. New newsletters will be automatically delivered to your Omnivore inbox.

-

NOTE: If Omnivore receive's an email that does not look like an article, such as a welcome message, or note from the author, it will be forwarded to your Omnivore account email address (the email you registered with).

-

Saving PDFs from a Mac 

-
    -
  1. -

    - Install the Mac App.  -

    -
    2. -
  2. -

    On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name.

    -
    3. -
  3. -

    - Select Share from the menu and choose Omnivore. -

    -
    4. -
  4. -

    The link will appear in your Library the next time you refresh it.

    -
    5. -
-

Reading

-

Click any link saved in your Library to enter the Reader view. 

-

Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load.

-

While reading, you can:

-
    -
  • -

    Change Formatting

    -
    • -
  • -

    Highlight Text

    -
    • -
  • -

    Add Notes

    -
    • -
  • -

    View All Saved Highlights and Notes

    -
    • -
  • -

    Track Reading Progress

    -
    • -
-

Change Formatting 

-
    -
  1. -

    - Theme: Tap your photo, initial, or avatar  in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme. -

    -
    2. -
  2. -

    - Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing. -

    -
    3. -
-

Highlight Text

-
    -
  1. -

    Select the text you wish to highlight.

    -
    2. -
  2. -

    - Tap the Highlight button. -

    -
    3. -
  3. -

    The text will appear highlighted next time you view the article.

    -
    4. -
-

Add Notes

-
    -
  1. -

    Highlight a section of text where you wish to add a note.

    -
    2. -
  2. -

    - Tap the Note button, type your note, and tap Save. -

    -
    3. -
  3. -

    The Note icon will appear next time you view this article.

    -
    4. -
-

View All Saved Highlights and Notes

-
    -
  1. -

    Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page.

    -
    2. -
  2. -

    To remove a note or highlight, select it from the list and tap the Trash icon.

    -
    3. -
-

Track Reading Progress

-

Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading.

-

Organizing

-

By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions: 

-
    -
  • -

    Archiving

    -
    • -
  • -

    Labels

    -
    • -
  • -

    Search

    -
    • -
  • -

    Filters

    -
    • -
-

Archiving

-
    -
  1. -

    Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu).

    -
    2. -
  2. -

    - Select Archive. -

    -
    3. -
  3. -

    The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below).

    -
    4. -
-

- Labels -

-
    -
  1. -

    - Tap the Menu icon next to any link and select Set Labels. -

    -
    2. -
  2. -

    - Select an existing label from the list or tap Edit Labels to create a new one. -

    -
    3. -
  3. -

    The label will appear next to the link in your Library. Tap it to view all links with the same label.

    -
    4. -
  4. -

    - Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label -

    -
    5. -
  5. -

    Note: Omnivore will automatically assign some labels, such as “Newsletters.”

    -
    6. -
-

Search

-
    -
  1. -

    To search through all your saved links, enter a keyword or phrase in the search bar. 

    -
    2. -
  2. -

    - You can combine keywords with labels and filters to focus your search even further. Learn more about advanced search. -

    -
    3. -
-

Filters

-
    -
  1. -

    - Use the Filters menu to refine your Library view (some filters may be visible by default). -

    -
    2. -
  2. -

    - Select Read Later to view a list of all your non-archived links except Newsletters. -

    -
    3. -
  3. -

    - Select Highlights to view the text selections you have highlighted in all your saved pages.  -

    -
    4. -
  4. -

    - Select Today to view a list of links you saved today. -

    -
    5. -
  5. -

    - Select Newsletters to view links saved via your newsletter subscriptions. -

    -
    6. -
-

Integrations

-

Omnivore allows integrations with knowledge bases and note-taking apps including:

-
    -
  • -

    Logseq

    -
    • -
  • -

    Webhooks

    -
    • -
-

Logseq

-

- With Omnivore's Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide. -

-

Webhooks

-

- Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive. -

-
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_get_started-original.html b/packages/api/src/services/popular_reads/omnivore_get_started-original.html deleted file mode 100644 index b536f4ccd..000000000 --- a/packages/api/src/services/popular_reads/omnivore_get_started-original.html +++ /dev/null @@ -1,1670 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Getting Started with Omnivore - Omnivore - - - - - - - - - - - - - - - - - - - - -
-
- -
-
-
-
-
- -
-

- Omnivore -

-
- -
-
-
-
-
-
-
- - - - -
-
-
- -
-
-
-
-
-
-
-
- TopNew - - - - - -
-
-
-
- -
-
- Learn the best ways to save links with Omnivore -
-
-
-
- Omnivore -
-
-
- -
-
- - - - -
- 2 -
- - - - - -
-
-
-
-
- -
-
- Highlighted <code> in Omnivore -
-
-
-
- Omnivore -
-
-
- -
-
- - - - -
- 1 -
- - - - - -
-
-
-
-
- -
-
- Add to your library with your Omnivore email address -
-
-
-
- Omnivore -
-
-
- -
-
- - - - -
- 1 -
- - - - - -
-
-
See all - - - - -
-
-
-
-
- -
- -
-
-
-
-
-
- © 2022 Omnivore -
-
- Privacy ∙ Terms ∙ Collection notice -
-
-
- - - - - - - Publish on Substack - - - - - Get the app -
-
- Substack is the home for great writing -
-
-
-
-
-
-
- - - - - - - diff --git a/packages/api/src/services/popular_reads/omnivore_ios-content.html b/packages/api/src/services/popular_reads/omnivore_ios-content.html deleted file mode 100644 index 2f79a0603..000000000 --- a/packages/api/src/services/popular_reads/omnivore_ios-content.html +++ /dev/null @@ -1,94 +0,0 @@ -
-
-
-

- With the Omnivore app for iOS, it’s easy to save web pages and articles or archive web content to read later. -

-

- The Omnivore app uses the iOS Share System, which lets you send items from one app (such as Safari) to another (such as Messages or Mail).  -

-
    -
  • -

    Step 1: Log in to the Omnivore app.

    -
    • -
  • -

    Step 2: Add Omnivore to your Share menu favorites.

    -
    • -
  • -

    Step 3: Save links to your Omnivore Library.

    -
    • -
-
-

- -

-
-

You must be logged in before you can save links via the Share menu. If you don’t already have an Omnivore account, you can sign up for free from the login screen.

-

- Note: If you haven’t installed the iOS app, download it here: https://omnivore.app/install/ios -

-

- Step 2: Add Omnivore to your Share menu favorites. -

-

Start by viewing the Share menu from within any supported iOS app (we’ve used Safari for this example).

-
    -
  1. -

    - Tap the Share icon at the bottom of the screen. -

    -
    2. -
  2. -

    - Swipe left to the end of the list of app icons and tap More. -

    -
    3. -
  3. -

    - Tap Edit at the top of the screen. -

    -
    4. -
  4. -

    - Scroll down until you see the Omnivore icon and tap the + icon next to it.  -

    -
    5. -
  5. -

    - Press and hold the three-bar icon and drag Omnivore to one of the top positions under Favorites. Tap Done to close the menu. -

    -
    6. -
  6. -

    - Omnivore will appear as one of the first options the next time you use the Share feature (you may need to restart Safari). -

    -
    7. -
-

- Step 3: Save links to your Omnivore Library -

-

- Start by navigating to the page or article you wish to save. Please note that Omnivore will save the content that appears on your screen (not just a link), so if the page is behind a paywall and you are logged into the paywalled site, you will save the paid content.  -

-
    -
  1. -

    - While viewing the page you’d like to save, tap the Share icon. -

    -
    2. -
  2. -

    - Tap the Omnivore icon in the Share menu. -

    -
    3. -
  3. -

    - Tag the article with one or more labels (optional) and tap Read Now or Read Later.  -

    -
    4. -
  4. -

    If you choose Read Later, the link will appear in your Library the next time you open the Omnivore app.

    -
    5. -
-
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_ios-original.html b/packages/api/src/services/popular_reads/omnivore_ios-original.html deleted file mode 100644 index 86daf6739..000000000 --- a/packages/api/src/services/popular_reads/omnivore_ios-original.html +++ /dev/null @@ -1,1456 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Saving Links from Your iPhone or iPad - Omnivore - - - - - - - - - - - - - - - - - - - - - -
-
- -
-
-
-
-
- -
-

- Omnivore -

-
- -
-
-
-
-
-
-
- - - - -
-
-
- -
-
-
-
-
-
- Comments -
-
-
-
-
-
- -
- -
-
-
-
- -
-
-
-
- - - -
-
-
-
-
-
-
-
-
- TopNew - - - - - -
-
-
-
- -
-
- Learn the best ways to save links with Omnivore -
-
-
-
- Omnivore -
-
-
- -
-
- - - - -
- 2 -
- - - - - -
-
-
-
-
- -
-
- Highlighted <code> in Omnivore -
-
-
-
- Omnivore -
-
-
- -
-
- - - - -
- 1 -
- - - - - -
-
-
-
-
- -
-
- Add to your library with your Omnivore email address -
-
-
-
- Omnivore -
-
-
- -
-
- - - - -
- 1 -
- - - - - -
-
-
See all - - - - -
-
-
-
-
- -
- -
-
-
-
-
-
- © 2022 Omnivore -
-
- Privacy ∙ Terms ∙ Collection notice -
-
-
- - - - - - - Publish on Substack - - - - - Get the app -
-
- Substack is the home for great writing -
-
-
-
-
-
-
- - - - - - - diff --git a/packages/api/src/services/popular_reads/omnivore_organize-content.html b/packages/api/src/services/popular_reads/omnivore_organize-content.html deleted file mode 100644 index b2ccf3e7c..000000000 --- a/packages/api/src/services/popular_reads/omnivore_organize-content.html +++ /dev/null @@ -1,73 +0,0 @@ -
-
-

Omnivore provides labels (also known as tags) to help you organize your library. Labels can be added to any saved read, and your library can be filtered based on labels.

-

On the web if you have a larger screen you can find the labels tool on the left side of the screen.

-
-
- - - - - -
Adding a label from the left menu
-
-
-

For a smaller screen you will find the labels button at the top of the page.

-
-
- - - - - -
Article Actions at the top of the Omnivore Reader for smaller screens
-
-
-

iOS users can long press on an item in the library or access the labels modal from the menu in the top right of the reader page.

-
-
- - - - - -
Editing labels from the library view on iOS
-
-
-

Label searches on iOS

-

On iOS you can use the label search modal to search for specific labels. This will create an OR search and return all links matching the assigned labels.

-
-
- - - - - -
Using labels to filter your search
-
-
-

Using Advanced Search to filter your library with labels

-

Omnivore's advanced search syntax supports searching for multiple labels using AND and OR clauses. You can also negate a label search to find pages that do not have a certain label.

-

Some examples:

-
    -
  • -

    - label:Newsletter finds all pages that have the label Newsletter -

    -
    • -
  • -

    - label:Cooking,Fitness finds all your pages with either the Cooking or Fitness labels -

    -
    • -
  • -

    - label:Newsletter label:Surfing finds all pages with both the Newsletter and Surfing labels -

    -
    • -
  • -

    label:Coding -label:News finds all pages with the Coding label that do not have the News label

    -
    • -
-
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/omnivore_organize-original.html b/packages/api/src/services/popular_reads/omnivore_organize-original.html deleted file mode 100644 index ff3036bc6..000000000 --- a/packages/api/src/services/popular_reads/omnivore_organize-original.html +++ /dev/null @@ -1,1022 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Organize your Omnivore library with labels - Omnivore - - - - - - - - - - - - - - - - - -
-
- -
-
-
-
-
- -
-

- Omnivore -

-
- -
-
-
-
-
-
-
- -
-
-
- -
-
-
-
-
-
-
- TopNew - - - - - -
-
-
-
- -
-
- Learn the best ways to save links with Omnivore -
-
- -
-
- -
-
- -
-
- - - - Share - - - - - - -
-
-
-
-
-
- -
-
- Highlighted <code> in Omnivore -
-
- -
-
- -
-
- -
-
- - - - Share - - - - - - -
-
-
-
-
-
- -
-
- Today we are happy to launch our new PDF viewer. It is available in our latest iOS release (1.3.0) and on the web. The new PDF viewer supports… -
-
- -
-
- -
-
- -
-
- - - - Share - - - - - - -
-
-
-
See all - - - - -
-
-
-
- -
-
-
-
-
-
-
- © 2022 Omnivore -
-
- Privacy ∙ Terms ∙ Collection notice -
-
-
- - - - - - - Publish on Substack - - - - - Get the app -
-
- Substack is the home for great writing -
-
-
-
-
-
- - - - - - - - diff --git a/packages/api/src/services/popular_reads/power_read_it_later-content.html b/packages/api/src/services/popular_reads/power_read_it_later-content.html deleted file mode 100644 index d8366c5f5..000000000 --- a/packages/api/src/services/popular_reads/power_read_it_later-content.html +++ /dev/null @@ -1,239 +0,0 @@ -
-
-
-
-
- -
Image via Nuno Cruz
-
-
-
-

- By Tiago Forte of Forte Labs -

-

At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year.

-

- -

-

This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask.

-

What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place.

-

We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours.

-

The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading.

-

The fact is, the ability to read is becoming a source of competitive advantage in the world.

-

I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time.

-

A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.”

-
-
-

-

Start Building Your Second Brain

-

Subscribe below to learn more about the next cohort of the Building a Second Brain course

-
-
-

Read It. Later.

-

Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar.

-

So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout.

-

On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are:

- -

The app I use, Pocket, adds a button to the Chrome toolbar that looks like this:

-
- -
Chrome toolbar
-
-

- Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. -

-

Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved:

-
-
- -
Mac desktop client
-
-

You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self:

-
-
- -
Web browser “tile” view
-
-

Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces).

-

The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development

-

I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity.

-

But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out.

-

There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with:

-
    -
  1. Increasing consumption of long-form content (which is presumably more substantive)
    2. -
  2. Better filtering
    3. -
-

#1 | Increasing Consumption of Long-Form Content

-

In order to consume good ideas, first you have to consume many ideas.

-

This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation.

-

The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances.

-
-

Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. -

-
-

Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them:

-

1. App performance

-

We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off.

-

If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal.

-

Meanwhile, your snaps and instas refresh at precog-like speeds.

-

Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about.

-

A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off.

-
- -
Effect of ad-blocker on loading times of Boston.com, via NYT -
-
-

Yeah that’s pretty much an eternity in mobile behavior land.

-

2. Matching content with your context

-
-
- -
My Pocket list on iPad
-
-
-

Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse.

-

So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in.

-

Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment.

-

David Allen puts it this way:

-
-

“It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.”

-
-

You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓

-

3. Asynchronous reading

-

This is one of the least understood barriers to reading in our fragmented timescape.

-

There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article.

-

The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it.

-

Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished.

-

4. Focus

-

A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it.

-

It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing.

-

Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read.

-
-

Actionable info ≠ Reference info ≠ To Read pile

-
-

Ergo,

-
-

Task manager ≠ Evernote ≠ Pocket

-
-

#2 | Better filtering

-

Now you’ve got the funnel filled. It’s time to narrow it.

-

Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources.

-

The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape.

-

Yeah.

-

In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you.

-

Procrastination is the most powerful force in the universe. It will find a way. -

-

I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time.

-

I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session.

-

So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab.

-

Here’s what a typical Monday morning link-fest looks like, just from email:

-

- -

-

Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing).

-

There’s only one rule: NO READING OR WATCHING! -

-

Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in.

-

I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about.

-

What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me.

-
-

Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide

-
-

Here’s a visual of how this works, from my Pocket analytics:

-

- -

-

You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week.

-

- /sidebar -

-

Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back.

-

I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on.

-

If only this method would scale.

-

- /end_sidebar -

-

Mo’ apps, mo’ problems

-

There are drawbacks, which I’ve glossed over until now. The two main ones:

-

1. Formatting issues

-

Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker.

-

The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed.

-

One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer.

-

2. Dependence

-

Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me.

-

For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved.

-

You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow:

-

- Highlight > Share > Share to Evernote > Save -

-

…has been replaced with this:

-

- Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket -

-

Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date.

-

- /rant_over -

-

Progress Traps and Paradigms

-

The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little.

-

As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know.

-

As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid.

-

What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go?

-

I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.”

-

When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens.

-

Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere.

-

But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on.

-

There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself.

-

Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain.

-

I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms.

-

I can’t put it any better than Donella Meadows, in her seminal piece on complex systems:

-
-

People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. -

-
-
-

It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia.

-
-
-

In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. -

-
-

Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own.

-

The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds.

-

And that is the secret power of Read It Later apps.

-

- P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) -

-
-

Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.

-
-
-

-

Join the Forte Labs Newsletter

-

Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I'll send you my Top 10 All-Time Articles right away as a thank you.

-
-
-
- -
-
\ No newline at end of file diff --git a/packages/api/src/services/popular_reads/power_read_it_later-original.html b/packages/api/src/services/popular_reads/power_read_it_later-original.html deleted file mode 100644 index 69ce2a783..000000000 --- a/packages/api/src/services/popular_reads/power_read_it_later-original.html +++ /dev/null @@ -1,1551 +0,0 @@ - - - - - - - - - - - - The Secret Power of ‘Read It Later’ Apps - Forte Labs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
-
-
-
-
-

- The Secret Power of ‘Read It Later’ Apps -

-
-
-
-
- -
-
-
-
-
    -
  • - Last modified -
    • -
  • - January 24, 2022 -
    • -
-
-
-
-
-
-

- Estimated reading time: 14 minutes -

-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
- Image via Nuno Cruz -
-
-
-
-

- By Tiago Forte of Forte Labs -

-

- At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year. -

-

- -

-

- This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask. -

-

- What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place. -

-

- We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours. -

-

- The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading. -

-

- The fact is, the ability to read is becoming a source of competitive advantage in the world. -

-

- I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time. -

-

- A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.” -

-
-
-
-
-
-

- Start Building Your Second Brain -

-
-
-

- Subscribe below to learn more about the next cohort of the Building a Second Brain course -

-
-
-
-
    -
    -
    - -
    - -
    -
    -
    -
    -

    -   -

    -

    - Read It. Later. -

    -

    - Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar. -

    -

    - So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout. -

    -

    - On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are: -

    - -

    - The app I use, Pocket, adds a button to the Chrome toolbar that looks like this: -

    -
    - -
    - Chrome toolbar -
    -
    -

    - Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. -

    -

    - Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved: -

    -
    -
    - -
    - Mac desktop client -
    -
    -

    - You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self: -

    -
    -
    - -
    - Web browser “tile” view -
    -
    -

    - Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces). -

    -

    - The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development -

    -

    - I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity. -

    -

    - But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out. -

    -

    - There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with: -

    -
      -
    1. Increasing consumption of long-form content (which is presumably more substantive) -
      2. -
    2. Better filtering -
      3. -
    -

    - #1 | Increasing Consumption of Long-Form Content -

    -

    - In order to consume good ideas, first you have to consume many ideas. -

    -

    - This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation. -

    -

    - The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances. -

    -
    -

    - Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. -

    -
    -

    - Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them: -

    -

    - 1. App performance -

    -

    - We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off. -

    -

    - If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal. -

    -

    - Meanwhile, your snaps and instas refresh at precog-like speeds. -

    -

    - Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about. -

    -

    - A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off. -

    -
    - -
    - Effect of ad-blocker on loading times of Boston.com, via NYT -
    -
    -

    - Yeah that’s pretty much an eternity in mobile behavior land. -

    -

    - 2. Matching content with your context -

    -
    -
    - -
    - My Pocket list on iPad -
    -
    -
    -

    - Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse. -

    -

    - So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in. -

    -

    - Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment. -

    -

    - David Allen puts it this way: -

    -
    -

    - “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.” -

    -
    -

    - You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓ -

    -

    - 3. Asynchronous reading -

    -

    - This is one of the least understood barriers to reading in our fragmented timescape. -

    -

    - There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article. -

    -

    - The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it. -

    -

    - Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished. -

    -

    - 4. Focus -

    -

    - A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it. -

    -

    - It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing. -

    -

    - Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read. -

    -
    -

    - Actionable info ≠ Reference info ≠ To Read pile -

    -
    -

    - Ergo, -

    -
    -

    - Task manager ≠ Evernote ≠ Pocket -

    -
    -

    - #2 | Better filtering -

    -

    - Now you’ve got the funnel filled. It’s time to narrow it. -

    -

    - Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources. -

    -

    - The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape. -

    -

    - Yeah. -

    -

    - In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you. -

    -

    - Procrastination is the most powerful force in the universe. It will find a way. -

    -

    - I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time. -

    -

    - I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session. -

    -

    - So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab. -

    -

    - Here’s what a typical Monday morning link-fest looks like, just from email: -

    -

    - -

    -

    - Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing). -

    -

    - There’s only one rule: NO READING OR WATCHING! -

    -

    - Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in. -

    -

    - I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about. -

    -

    - What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me. -

    -
    -

    - Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide -

    -
    -

    - Here’s a visual of how this works, from my Pocket analytics: -

    -

    - -

    -

    - You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week. -

    -

    - /sidebar -

    -

    - Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back. -

    -

    - I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on. -

    -

    - If only this method would scale. -

    -

    - /end_sidebar -

    -

    - Mo’ apps, mo’ problems -

    -

    - There are drawbacks, which I’ve glossed over until now. The two main ones: -

    -

    - 1. Formatting issues -

    -

    - Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker. -

    -

    - The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed. -

    -

    - One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer. -

    -

    - 2. Dependence -

    -

    - Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me. -

    -

    - For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved. -

    -

    - You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow: -

    -

    - Highlight > Share > Share to Evernote > Save -

    -

    - …has been replaced with this: -

    -

    - Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket -

    -

    - Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date. -

    -

    - /rant_over -

    -

    - Progress Traps and Paradigms -

    -

    - The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little. -

    -

    - As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know. -

    -

    - As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid. -

    -

    - What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go? -

    -

    - I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.” -

    -

    - When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens. -

    -

    - Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere. -

    -

    - But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on. -

    -

    - There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself. -

    -

    - Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain. -

    -

    - I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms. -

    -

    - I can’t put it any better than Donella Meadows, in her seminal piece on complex systems: -

    -
    -

    - People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. -

    -
    -
    -

    - It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia. -

    -
    -
    -

    - In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. -

    -
    -

    - Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own. -

    -

    - The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds. -

    -

    - And that is the secret power of Read It Later apps. -

    -

    - P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) -

    -
    - Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.
    -
    -
    -
    -
    -
    -
    -
    -

    - Join the Forte Labs Newsletter -

    -
    -
    -

    - Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I'll send you my Top 10 All-Time Articles right away as a thank you. -

    -
    -
    -
    -
      -
      -
      - -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -

      - Get my top-10 productivity tips -

      -
      -
      -
      -
      -
      - Subscribe to my free weekly newsletter and get my best tips on how to use technology to be more productive. -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - RESOURCES -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - ABOUT -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - CONNECT -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      - Twitter Facebook Youtube Linkedin Instagram -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - - -
      -
        -
      • - -
        • -
      • - -
        • -
      • - -
        • -
      • - -
        • -
      • - -
        • -
      • - -
        • -
      -
      -
      -
      -
      - Share This -
      - -
      -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/api/src/services/popular_reads/rlove_carnitas-content.html b/packages/api/src/services/popular_reads/rlove_carnitas-content.html deleted file mode 100644 index a02fdb0d6..000000000 --- a/packages/api/src/services/popular_reads/rlove_carnitas-content.html +++ /dev/null @@ -1,34 +0,0 @@ -
      -
      -
      -
      -

      I used to have a bunch of recipes up online. But writing recipes is no fun; it is difficult to capture the beauty of a dish with a bunch of steps. Moreover, using recipes isn’t how I cook. I want to understand the flavors of a dish and then execute it in my own way, in my own hands. So the recipes went away.

      -

      But one of the most popular — and one of my personal favorites — was a recipe for the Mexican pork dish carnitas. It was a fun, relatively easy recipe, not traditional in approach but fairly traditional (and really delicious) in output. Folks keep asking for it. So here it is, in hopes I can eat it at your next house party.

      -

      Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. The traditional recipe is simple: several pounds of pork shoulder, a pound or two of lard, orange peel, and some water (or coca-cola), slow roasted and then “boiled” to a crisp. That is…a bit much. What follows is not an authentic approach.

      -
      -

      -

      -
      Photo by flickr user mccun934 licensed CC BY 2.0 -
      -
      -

      Never ones to eschew taste for health, the French have nonetheless taught us nothing if not that we can make a succulent, flavorful dish without boiling a tough cut of meat in lard. In that vein, my recipe is more of a braiser de porc than a confit de porc — pork shoulder, aromatics, citrus juice, and a little Grand Marnier, slow cooked on the stove. Healthier for the heart, but also — more importantly — tastier to the tongue. To obtain that classic carnitas crisp, we braise the meat uncovered until the liquid evaporates and then move the pot to the oven and caramelize.

      -

      Makes about 6 servings.

      -

      Ingredients:

      -

      3 lbs boneless pork shoulder, preferably Boston butt (the upper shoulder)
      4 tablespoons grape seed oil
      6 cloves garlic, peeled and thinly-sliced
      2 oranges, juiced, plus the zest of 1/2
      2 limes, juiced and zested
      several cups chicken stock
      1 teaspoon ground cayenne pepper
      1 tablespoon ground ancho chile
      1 tablespoon ground chipotle pepper
      2 teaspoons cumin
      1 teaspoon ground cinnamon
      1 teaspoon freshly-ground black pepper
      - herb sachet with 8 sprigs Mexican oregano, 6 sprigs thyme, 2 sticks cinnamon, and 2 bay leaves, wrapped in a cheese cloth and tied shut with cooking twine
      1 cup Grand Marnier
      coarse sea salt, to coat meat, plus more to taste -

      -

      Cut the pork shoulder into 5" chunks. Remove any gratuitously-excessive fat, but leave at least a thin layer. Sprinkle the chunks with sea salt. Let sit at room temperature at least 30 minutes. Pat dry.

      -

      Heat the grape seed oil in a dutch oven over medium-high heat. Sauté the pork shoulder until well browned on each side. If needed, sauté across multiple batches.

      -

      Add the garlic and sauté for one more minute.

      -

      Add the orange zest, lime zest, ground cayenne pepper, ground ancho chile, ground chipotle chile, cumin, ground cinnamon, black pepper, and herb sachet. Mix.

      -

      Add the Grand Marnier, orange juice, and lime juice. Stir, scrapping the bottom of the pot.

      -

      Add chicken stock as needed such that the pork is two-thirds submerged in liquid.

      -

      Stirring, raise heat to high and bring to a boil. Once boiling, lower heat until the liquid is at a simmer and braise, uncovered, stirring occasionally, until the pork is cooked and tender but not disintegrating and the liquid is reduced by at least two-thirds, about three hours. If the liquid gets perilously-low while cooking, add a little chicken stock.

      -

      Remove pork from pot. Once cool enough to handle, use a fork to shred the pork into bite-sized, but fairly large, chunks, removing any fatty pieces as desired.

      -

      Preheat oven to 450°F.

      -

      Return pork chunks to pot. Place uncovered pot in oven. Continue cooking until the liquid has evaporated and the pork is crispy and starting to caramelize, about 20 minutes.

      -

      Taste and adjust salt. Serve with warm corn tortillas, guacamole, pico de gallo, diced white onion, chopped cilantro, lime wedges, margaritas, and college football.

      -
      -
      -
      -
      \ No newline at end of file diff --git a/packages/api/src/services/popular_reads/rlove_carnitas-original.html b/packages/api/src/services/popular_reads/rlove_carnitas-original.html deleted file mode 100644 index e4db70b00..000000000 --- a/packages/api/src/services/popular_reads/rlove_carnitas-original.html +++ /dev/null @@ -1,1230 +0,0 @@ - - - - - Slow-Braised Carnitas Recipe. Robert Love’s recipe for carnitas, in… | by Robert Love | Medium - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      -
      -
      - -
      - -
      -
      -
      -
      -
      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      - Robert Love -
      -
      -
      -
      -
      -
      -
      - -
      -
      - -
      -
      -
      -
      -

      - Feb 24, 2017 -

      - -
      - 3 min read -
      - -
      -
      -
      -
      -
      - -

      - -

      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      - -
      -
      -
      -
      - -
      -
      -
      -
      - -
      -
      -
      - -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - -

      - -

      -
      -
      -
      - -
      -
      -
      -
      - -
      -
      -
      -
      - -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -

      - Slow-Braised Carnitas Recipe -

      -
      - - - -
      -
      - -
      -
      - Photo by flickr user mccun934 licensed CC BY 2.0 -
      -
      - - - - - - - - - - - - - - - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      - - -
      -
      -
      -
      - -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      - - -
      -
      -
      -
      - -
      -
      -
      -
      -
      - -
      - - -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -

      - More from Robert Love -

      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -

      - Google, Android, Linux kernel, author, Boston -

      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -

      - Love podcasts or audiobooks? Learn on the go with our new app. -

      -
      -
      - Try Knowable -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - - -
      -

      - AboutHelpTermsPrivacy -

      -
      -
      - -

      - Get the Medium app -

      -
      -
      - A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store -
      A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      - -
      -
      -
      -
      -
      - -
      -
      -
      - Robert Love -
      -
      -
      -

      - Robert Love -

      -
      -
      -

      - Google, Android, Linux kernel, author, Boston -

      -
      -
      - -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      -

      - More from Medium -

      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - Back2Life -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -

      - This Produce Season: Winter -

      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - Jeannie @ Paw It Forward -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -

      - Eli’s Level 2 Board & Train: Small Dog with Big Dog Syndrome -

      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -
      -
      -
      - Amit Sarda -
      -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -
      -

      - in -

      -
      -
      - -

      - Simple Cooking -

      -
      -
      -
      -

      - Masala Chai Recipe -

      -
      -
      -
      -
      -
      - A cup of masala chai (Indian spiced tea) in a green plate -
      -
      -
      -
      -
      - -
      - -
      -
      -
      -
      -
      -
      - Australian Foodie -
      -
      -
      -
      -
      -
      -
      -
      - -
      -
      -

      - in -

      -
      -
      - -

      - Global Foodies -

      -
      -
      -

      - Vegemite Spaghetti Recipe — An Easy One Pot Dinner -

      -
      -
      -
      -
      - Vegemite pasta spaghetti recipe -
      -
      -
      -
      -
      - -

      - Help -

      -
      -
      - -

      - Status -

      -
      -
      - -

      - Writers -

      -
      -
      - -

      - Blog -

      -
      -
      - -

      - Careers -

      -
      -
      - -

      - Privacy -

      -
      -
      - -

      - Terms -

      -
      -
      - -

      - About -

      -
      -
      - -

      - Knowable -

      -
      -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index ad8d6d81c..4f436c5fe 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -1,12 +1,12 @@ import { Readability } from '@omnivore/readability' import { DeepPartial } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' +import { Highlight } from '../entity/highlight' import { LibraryItem, LibraryItemState } from '../entity/library_item' import { User } from '../entity/user' import { homePageURL } from '../env' import { ArticleSavingRequestStatus, - HighlightType, Maybe, PreparedDocumentInput, SaveErrorCode, @@ -36,6 +36,7 @@ const FORCE_PUPPETEER_URLS = [ TWEET_URL_REGEX, /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w-]+\?v=|embed\/|v\/)?)([\w-]+)(\S+)?$/, ] +const ALREADY_PARSED_SOURCES = ['puppeteer-parse', 'csv-importer', 'rss-feeder'] const createSlug = (url: string, title?: Maybe | undefined) => { const { pathname } = new URL(url) @@ -52,7 +53,7 @@ const createSlug = (url: string, title?: Maybe | undefined) => { const shouldParseInBackend = (input: SavePageInput): boolean => { return ( - input.source !== 'puppeteer-parse' && + ALREADY_PARSED_SOURCES.indexOf(input.source) === -1 && FORCE_PUPPETEER_URLS.some((regex) => regex.test(input.url)) ) } @@ -77,6 +78,7 @@ export const savePage = async ( let clientRequestId = input.clientRequestId const itemToSave = parsedContentToLibraryItem({ + itemId: clientRequestId, url: input.url, title: input.title, userId: user.id, @@ -99,7 +101,7 @@ export const savePage = async ( await createPageSaveRequest({ userId: user.id, url: itemToSave.originalUrl, - articleSavingRequestId: clientRequestId, + articleSavingRequestId: clientRequestId || undefined, state: input.state || undefined, labels: input.labels || undefined, }) @@ -118,6 +120,9 @@ export const savePage = async ( }) ) if (existingLibraryItem) { + clientRequestId = existingLibraryItem.id + slug = existingLibraryItem.slug + // we don't want to update an rss feed item if rss-feeder is tring to re-save it if (existingLibraryItem.subscription === input.rssFeedUrl) { return { @@ -126,16 +131,24 @@ export const savePage = async ( } } - clientRequestId = existingLibraryItem.id - slug = existingLibraryItem.slug + // update the item except for id and slug await updateLibraryItem( clientRequestId, - itemToSave as QueryDeepPartialEntity, + { + ...itemToSave, + id: undefined, + slug: undefined, + } as QueryDeepPartialEntity, user.id ) } else { // do not publish a pubsub event if the item is imported - const newItem = await createLibraryItem(itemToSave, user.id) + const newItem = await createLibraryItem( + itemToSave, + user.id, + undefined, + isImported + ) clientRequestId = newItem.id } @@ -158,12 +171,10 @@ export const savePage = async ( } if (parseResult.highlightData) { - const highlight = { - updatedAt: new Date(), - createdAt: new Date(), - userId: user.id, + const highlight: DeepPartial = { ...parseResult.highlightData, - type: HighlightType.Highlight, + user: { id: user.id }, + libraryItem: { id: clientRequestId }, } if (!(await createHighlight(highlight, clientRequestId, user.id))) { @@ -241,7 +252,7 @@ export const parsedContentToLibraryItem = ({ publishedAt: validatedDate( publishedAt || parsedContent?.publishedDate || undefined ), - uploadFile: { id: uploadFileId ?? undefined }, + uploadFileId: uploadFileId || undefined, readingProgressTopPercent: 0, readingProgressHighestReadAnchor: 0, state: state @@ -255,5 +266,7 @@ export const parsedContentToLibraryItem = ({ wordCount: wordsCount(parsedContent?.textContent || ''), contentReader: contentReaderForLibraryItem(itemType, uploadFileId), subscription: rssFeedUrl, + archivedAt: + state === ArticleSavingRequestStatus.Archived ? new Date() : undefined, } } diff --git a/packages/api/src/services/save_url.ts b/packages/api/src/services/save_url.ts index 0f14b1658..3165bb8fb 100644 --- a/packages/api/src/services/save_url.ts +++ b/packages/api/src/services/save_url.ts @@ -42,9 +42,7 @@ export const saveUrlFromEmail = async ( clientRequestId: string, userId: string ): Promise => { - const user = await userRepository.findOneBy({ - id: userId, - }) + const user = await userRepository.findById(userId) if (!user) { return false } diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts index 8aaea89b8..fcf25af01 100644 --- a/packages/api/src/services/subscriptions.ts +++ b/packages/api/src/services/subscriptions.ts @@ -1,4 +1,5 @@ import axios from 'axios' +import { DeepPartial, DeleteResult } from 'typeorm' import { appDataSource } from '../data_source' import { NewsletterEmail } from '../entity/newsletter_email' import { Subscription } from '../entity/subscription' @@ -189,7 +190,8 @@ export const createSubscription = async ( newsletterEmail?: NewsletterEmail, status = SubscriptionStatus.Active, unsubscribeMailTo?: string, - subscriptionType = SubscriptionType.Newsletter + subscriptionType = SubscriptionType.Newsletter, + url?: string ): Promise => { return getRepository(Subscription).save({ user: { id: userId }, @@ -199,5 +201,16 @@ export const createSubscription = async ( unsubscribeMailTo, lastFetchedAt: new Date(), type: subscriptionType, + url, }) } + +export const deleteSubscription = async (id: string): Promise => { + return getRepository(Subscription).delete(id) +} + +export const createRssSubscriptions = async ( + subscriptions: DeepPartial[] +) => { + return getRepository(Subscription).save(subscriptions) +} diff --git a/packages/api/src/services/user.ts b/packages/api/src/services/user.ts index aed61b8f9..870d9ce6b 100644 --- a/packages/api/src/services/user.ts +++ b/packages/api/src/services/user.ts @@ -1,4 +1,4 @@ -import { User } from '../entity/user' +import { StatusType, User } from '../entity/user' import { authTrx } from '../repository' import { userRepository } from '../repository/user' @@ -13,7 +13,7 @@ export const deleteUser = async (userId: string) => { } export const updateUser = async (userId: string, update: Partial) => { - await authTrx( + return authTrx( async (t) => t.getRepository(User).update(userId, update), undefined, userId @@ -21,5 +21,5 @@ export const updateUser = async (userId: string, update: Partial) => { } export const findUser = async (id: string): Promise => { - return userRepository.findOneBy({ id }) + return userRepository.findOneBy({ id, status: StatusType.Active }) } diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 71df1cbc5..5764307a2 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -21,6 +21,7 @@ interface BackendEnv { gateway_url: string apiEnv: string instanceId: string + trustProxy: boolean } client: { url: string @@ -159,6 +160,7 @@ const nullableEnvVars = [ 'RSS_FEED_TASK_HANDLER_URL', 'SENDGRID_VERIFICATION_TEMPLATE_ID', 'REMINDER_TASK_HANDLER_URL', + 'TRUST_PROXY', ] // Allow some vars to be null/empty /* If not in GAE and Prod/QA/Demo env (f.e. on localhost/dev env), allow following env vars to be null */ @@ -207,6 +209,7 @@ export function getEnv(): BackendEnv { apiEnv: parse('API_ENV'), instanceId: parse('GAE_INSTANCE') || `x${os.userInfo().username}_${os.hostname()}`, + trustProxy: parse('TRUST_PROXY') === 'true', } const client = { url: parse('CLIENT_URL'), diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index bb3457e00..f6f2ac6dc 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -7,7 +7,6 @@ import axios from 'axios' import { nanoid } from 'nanoid' import { DeepPartial } from 'typeorm' import { Recommendation } from '../entity/recommendation' -import { Subscription } from '../entity/subscription' import { env } from '../env' import { ArticleSavingRequestStatus, @@ -592,19 +591,30 @@ export const enqueueThumbnailTask = async ( return createdTasks[0].name } -export const enqueueRssFeedFetch = async ( - userId: string, - rssFeedSubscription: Subscription -): Promise => { - const { GOOGLE_CLOUD_PROJECT } = process.env - const payload = { - subscriptionId: rssFeedSubscription.id, - feedUrl: rssFeedSubscription.url, - lastFetchedAt: rssFeedSubscription.lastFetchedAt?.getTime() || 0, // unix timestamp in milliseconds - } +export interface RssSubscriptionGroup { + url: string + subscriptionIds: string[] + userIds: string[] + fetchedDates: (Date | null)[] + scheduledDates: Date[] + checksums: (string | null)[] +} - const headers = { - [OmnivoreAuthorizationHeader]: generateVerificationToken({ id: userId }), +export const enqueueRssFeedFetch = async ( + subscriptionGroup: RssSubscriptionGroup +): Promise => { + const { GOOGLE_CLOUD_PROJECT, PUBSUB_VERIFICATION_TOKEN } = process.env + const payload = { + subscriptionIds: subscriptionGroup.subscriptionIds, + feedUrl: subscriptionGroup.url, + lastFetchedTimestamps: subscriptionGroup.fetchedDates.map( + (timestamp) => timestamp?.getTime() || 0 + ), // unix timestamp in milliseconds + lastFetchedChecksums: subscriptionGroup.checksums, + scheduledTimestamps: subscriptionGroup.scheduledDates.map((timestamp) => + timestamp.getTime() + ), // unix timestamp in milliseconds + userIds: subscriptionGroup.userIds, } // If there is no Google Cloud Project Id exposed, it means that we are in local environment @@ -613,9 +623,10 @@ export const enqueueRssFeedFetch = async ( // Calling the handler function directly. setTimeout(() => { axios - .post(env.queue.rssFeedTaskHandlerUrl, payload, { - headers, - }) + .post( + `${env.queue.rssFeedTaskHandlerUrl}?token=${PUBSUB_VERIFICATION_TOKEN}`, + payload + ) .catch((error) => { logError(error) }) @@ -628,8 +639,7 @@ export const enqueueRssFeedFetch = async ( project: GOOGLE_CLOUD_PROJECT, queue: 'omnivore-rss-queue', payload, - taskHandlerUrl: env.queue.rssFeedTaskHandlerUrl, - requestHeaders: headers, + taskHandlerUrl: `${env.queue.rssFeedTaskHandlerUrl}?token=${PUBSUB_VERIFICATION_TOKEN}`, }) if (!createdTasks || !createdTasks[0].name) { diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index dadcfe171..bdfe443c1 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -268,7 +268,6 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ ), image: item.thumbnail, highlights: item.highlights?.map(highlightDataToHighlight), - uploadFileId: item.uploadFile?.id, wordsCount: item.wordCount, }) diff --git a/packages/api/src/utils/parser.ts b/packages/api/src/utils/parser.ts index ae047ef82..eb8ece9c6 100644 --- a/packages/api/src/utils/parser.ts +++ b/packages/api/src/utils/parser.ts @@ -16,6 +16,7 @@ import { ILike } from 'typeorm' import { promisify } from 'util' import { v4 as uuid } from 'uuid' import { Highlight } from '../entity/highlight' +import { StatusType } from '../entity/user' import { env } from '../env' import { PageType, PreparedDocumentInput } from '../generated/graphql' import { userRepository } from '../repository/user' @@ -465,6 +466,7 @@ export const isProbablyArticle = async ( ): Promise => { const user = await userRepository.findOneBy({ email: ILike(email), + status: StatusType.Active, }) return !!user || subject.includes(ARTICLE_PREFIX) } diff --git a/packages/api/src/utils/search.ts b/packages/api/src/utils/search.ts index c4799ba7b..96abd859b 100644 --- a/packages/api/src/utils/search.ts +++ b/packages/api/src/utils/search.ts @@ -249,13 +249,13 @@ const parseDateFilter = ( switch (field.toUpperCase()) { case 'PUBLISHED': - field = 'publishedAt' + field = 'published_at' break case 'SAVED': - field = 'savedAt' + field = 'saved_at' break case 'UPDATED': - field = 'updatedAt' + field = 'updated_at' } return { diff --git a/packages/api/src/utils/uploads.ts b/packages/api/src/utils/uploads.ts index 34bfb5854..2d31b525f 100644 --- a/packages/api/src/utils/uploads.ts +++ b/packages/api/src/utils/uploads.ts @@ -34,10 +34,6 @@ const storage = env.fileUpload?.gcsUploadSAKeyFilePath : new Storage() const bucketName = env.fileUpload.gcsUploadBucket -export const getFilePublicUrl = (filePathName: string): string => { - return storage.bucket(bucketName).file(filePathName).publicUrl() -} - export const countOfFilesWithPrefix = async (prefix: string) => { const [files] = await storage.bucket(bucketName).getFiles({ prefix }) return files.length @@ -81,22 +77,6 @@ export const generateDownloadSignedUrl = async ( return url } -export const makeStorageFilePublic = async ( - id: string, - fileName: string -): Promise => { - // if (env.dev.isLocal) { - // return 'http://localhost:3000/public/' + id + '/' + fileName - // } - - // Makes the file public - const filePathName = generateUploadFilePathName(id, fileName) - await storage.bucket(bucketName).file(filePathName).makePublic() - - const fileObj = storage.bucket(bucketName).file(filePathName) - return fileObj.publicUrl() -} - export const getStorageFileDetails = async ( id: string, fileName: string diff --git a/packages/api/test/resolvers/article.test.ts b/packages/api/test/resolvers/article.test.ts index e3b13c0d1..3b6e80057 100644 --- a/packages/api/test/resolvers/article.test.ts +++ b/packages/api/test/resolvers/article.test.ts @@ -4,6 +4,7 @@ import chaiString from 'chai-string' import 'mocha' import sinon from 'sinon' import { DeepPartial } from 'typeorm' +import { Group } from '../../src/entity/groups/group' import { Highlight } from '../../src/entity/highlight' import { Label } from '../../src/entity/label' import { LibraryItem, LibraryItemState } from '../../src/entity/library_item' @@ -15,14 +16,15 @@ import { PageType, SyncUpdatedItemEdge, UpdateReason, - UploadFileStatus, + UploadFileStatus } from '../../src/generated/graphql' import { getRepository } from '../../src/repository' +import { createGroup, deleteGroup } from '../../src/services/groups' import { createHighlight } from '../../src/services/highlights' import { createLabel, deleteLabels, - saveLabelsInLibraryItem, + saveLabelsInLibraryItem } from '../../src/services/labels' import { createLibraryItem, @@ -33,7 +35,7 @@ import { deleteLibraryItemsByUserId, findLibraryItemById, findLibraryItemByUrl, - updateLibraryItem, + updateLibraryItem } from '../../src/services/library_item' import { deleteUser } from '../../src/services/user' import * as createTask from '../../src/utils/createTask' @@ -153,6 +155,13 @@ const searchQuery = (keyword = '') => { highlights { id } + labels { + id + name + } + recommendations { + name + } } } pageInfo { @@ -176,15 +185,17 @@ const savePageQuery = ( title: string, originalContent: string, state: ArticleSavingRequestStatus | null = null, - labels: string[] | null = null + labels: string[] | null = null, + clientRequestId = generateFakeUuid(), + source = 'puppeteer-parse' ) => { return ` mutation { savePage( input: { url: "${url}", - source: "test", - clientRequestId: "${generateFakeUuid()}", + source: "${source}", + clientRequestId: "${clientRequestId}", title: "${title}", originalContent: "${originalContent}" state: ${state} @@ -284,16 +295,18 @@ const setBookmarkQuery = (articleId: string, bookmark: boolean) => { const saveArticleReadingProgressQuery = ( articleId: string, progress: number, - topPercent: number | null = null + topPercent: number | null = null, + force: boolean | null = null ) => { return ` mutation { saveArticleReadingProgress( input: { id: "${articleId}", - readingProgressPercent: ${progress} - readingProgressAnchorIndex: 0 - readingProgressTopPercent: ${topPercent} + readingProgressPercent: ${progress}, + readingProgressAnchorIndex: 0, + readingProgressTopPercent: ${topPercent}, + force: ${force} } ) { ... on SaveArticleReadingProgressSuccess { @@ -568,22 +581,26 @@ describe('Article API', () => { // Now save the link again, and ensure it is returned await graphqlRequest( - savePageQuery(url, title, originalContent), + savePageQuery(url, title, originalContent, null, null, generateFakeUuid()), authToken ).expect(200) allLinks = await graphqlRequest(searchQuery(''), authToken).expect(200) + expect(allLinks.body.data.search.edges[0].node.id).to.eq(justSavedId) expect(allLinks.body.data.search.edges[0].node.url).to.eq(url) }) }) - xcontext('when we also want to save labels and archives the item', () => { + context('when we also want to save labels and archives the item', () => { + before(() => { + url = 'https://blog.omnivore.app/new-url-2' + }) + after(async () => { - await deleteLibraryItemById(url, user.id) + await deleteLibraryItemByUrl(url, user.id) }) it('saves the labels and archives the item', async () => { - url = 'https://blog.omnivore.app/new-url-2' const state = ArticleSavingRequestStatus.Archived const labels = ['test name', 'test name 2'] await graphqlRequest( @@ -596,6 +613,29 @@ describe('Article API', () => { expect(savedItem?.labels?.map((l) => l.name)).to.eql(labels) }) }) + + context('when the source is rss-feeder and url is from youtube.com', () => { + const source = 'rss-feeder' + const stub = sinon.stub(createTask, 'enqueueParseRequest') + + before(() => { + url = 'https://www.youtube.com/watch?v=123' + }) + + after(async () => { + await deleteLibraryItemByUrl(url, user.id) + sinon.restore() + }) + + it('does not parse in the backend', async () => { + await graphqlRequest( + savePageQuery(url, title, originalContent, null, null, '', source), + authToken + ).expect(200) + + expect(stub).not.to.have.been.called + }) + }) }) describe('SaveUrl', () => { @@ -626,22 +666,6 @@ describe('Article API', () => { ) }) }) - - xcontext('when we save labels', () => { - it('saves the labels and archives the item', async () => { - url = 'https://blog.omnivore.app/new-url-2' - const state = ArticleSavingRequestStatus.Archived - const labels = ['test name', 'test name 2'] - await graphqlRequest( - saveUrlQuery(url, state, labels), - authToken - ).expect(200) - - const savedItem = await findLibraryItemByUrl(url, user.id) - expect(savedItem?.archivedAt).to.not.be.null - expect(savedItem?.labels?.map((l) => l.name)).to.eql(labels) - }) - }) }) describe('setBookmarkArticle', () => { @@ -706,7 +730,6 @@ describe('Article API', () => { ).to.eq(75) // Now try to set to a lower value (50), value should not be updated - // refresh index to ensure the reading progress is updated const secondQuery = saveArticleReadingProgressQuery(itemId, 50) const secondRes = await graphqlRequest(secondQuery, authToken).expect(200) expect( @@ -751,6 +774,37 @@ describe('Article API', () => { 'BAD_DATA', ]) }) + + context('when force is true', () => { + before(async () => { + itemId = (await createLibraryItem({ + user: { id: user.id }, + originalUrl: 'https://blog.omnivore.app/setBookmarkArticle', + slug: 'test-with-omnivore', + readableContent: '

      test

      ', + title: 'test title', + readingProgressBottomPercent: 100, + readingProgressTopPercent: 80, + }, user.id)).id + }) + + after(async () => { + await deleteLibraryItemById(itemId, user.id) + }) + + it('ignore position check if force is true', async () => { + query = saveArticleReadingProgressQuery(itemId, 20, 10, true) + const res = await graphqlRequest(query, authToken).expect(200) + expect( + res.body.data.saveArticleReadingProgress.updatedArticle + .readingProgressPercent + ).to.eql(20) + expect( + res.body.data.saveArticleReadingProgress.updatedArticle + .readingProgressTopPercent + ).to.eql(10) + }) + }) }) describe('SaveFile', () => { @@ -1456,6 +1510,64 @@ describe('Article API', () => { }) } ) + + context('when recommendedBy:* is in the query', () => { + let items: LibraryItem[] = [] + let group: Group + + before(async () => { + keyword = 'recommendedBy:*' + + group = ( + await createGroup({ + admin: user, + name: 'test group', + }) + )[0] + + // Create some test items + items = await createLibraryItems( + [ + { + user, + title: 'test title 1', + readableContent: '

      test 1

      ', + slug: 'test slug 1', + originalUrl: `${url}/test1`, + recommendations: [ + { + recommender: user, + group, + }, + ], + }, + { + user, + title: 'test title 2', + readableContent: '

      test 2

      ', + slug: 'test slug 2', + originalUrl: `${url}/test2`, + }, + ], + user.id + ) + }) + + after(async () => { + await deleteLibraryItems(items, user.id) + await deleteGroup(group.id) + }) + + it('returns recommended items', async () => { + const res = await graphqlRequest(query, authToken).expect(200) + + expect(res.body.data.search.pageInfo.totalCount).to.eq(1) + expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id) + expect( + res.body.data.search.edges[0].node.recommendations[0].name + ).to.eq(group.name) + }) + }) }) describe('TypeaheadSearch API', () => { @@ -1591,6 +1703,21 @@ describe('Article API', () => { UpdateReason.Deleted ) }) + + context('when since is -1000000000-01-01T00:00:00Z from android app', () => { + before(() => { + since = '-1000000000-01-01T00:00:00Z' + }) + + it('returns all', async () => { + const res = await graphqlRequest( + updatesSinceQuery(since), + authToken + ).expect(200) + + expect(res.body.data.updatesSince.edges.length).to.eql(5) + }) + }) }) describe('BulkAction API', () => { @@ -1647,18 +1774,56 @@ describe('Article API', () => { }) }) - context('when action is Archive', () => { - it('archives all items', async () => { - const res = await graphqlRequest( - bulkActionQuery(BulkActionType.Archive), - authToken - ).expect(200) - expect(res.body.data.bulkAction.success).to.be.true + context( + 'when action is Archive and query is published:*..2023-10-01', + () => { + let items: LibraryItem[] = [] - const items = await graphqlRequest(searchQuery(), authToken).expect(200) - expect(items.body.data.search.pageInfo.totalCount).to.eql(0) - }) - }) + before(async () => { + items = await createLibraryItems( + [ + { + user, + title: 'test item', + readableContent: '

      test

      ', + slug: 'test-item', + originalUrl: `https://blog.omnivore.app/p/bulk-action-archive`, + publishedAt: new Date('2023-10-01'), + }, + { + user, + title: 'test item 2', + readableContent: '

      test

      ', + slug: 'test-item-2', + originalUrl: `https://blog.omnivore.app/p/bulk-action-archive-2`, + publishedAt: new Date('2023-10-02'), + }, + ], + user.id + ) + }) + + after(async () => { + // Delete all items + await deleteLibraryItems(items, user.id) + }) + + it('archives old items', async () => { + const res = await graphqlRequest( + bulkActionQuery(BulkActionType.Archive, 'published:*..2023-10-01'), + authToken + ).expect(200) + expect(res.body.data.bulkAction.success).to.be.true + + const response = await graphqlRequest( + searchQuery('in:archive'), + authToken + ).expect(200) + expect(response.body.data.search.pageInfo.totalCount).to.eql(1) + expect(response.body.data.search.edges[0].node.id).to.eql(items[0].id) + }) + } + ) context('when action is Delete', () => { it('deletes all items', async () => { diff --git a/packages/api/test/resolvers/subscriptions.test.ts b/packages/api/test/resolvers/subscriptions.test.ts index 07393fbdb..02f7cd1b8 100644 --- a/packages/api/test/resolvers/subscriptions.test.ts +++ b/packages/api/test/resolvers/subscriptions.test.ts @@ -1,5 +1,6 @@ import chai, { expect } from 'chai' import 'mocha' +import Parser from 'rss-parser' import sinon from 'sinon' import sinonChai from 'sinon-chai' import { NewsletterEmail } from '../../src/entity/newsletter_email' @@ -9,13 +10,13 @@ import { SubscriptionStatus, SubscriptionType, } from '../../src/generated/graphql' -import { - createSubscription, - unsubscribe, - UNSUBSCRIBE_EMAIL_TEXT -} from '../../src/services/subscriptions' import { getRepository } from '../../src/repository' import { createNewsletterEmail } from '../../src/services/newsletters' +import { + createSubscription, + deleteSubscription, + UNSUBSCRIBE_EMAIL_TEXT, +} from '../../src/services/subscriptions' import { deleteUser } from '../../src/services/user' import * as sendEmail from '../../src/utils/sendEmail' import { createTestUser } from '../db' @@ -146,7 +147,7 @@ describe('Subscriptions API', () => { undefined, SubscriptionType.Newsletter ) - const allSubscriptions = [sub5, ...subscriptions] + const allSubscriptions = [...subscriptions, sub5] const res = await graphqlRequest(query, authToken).expect(200) expect(res.body.data.subscriptions.subscriptions).to.eql( @@ -347,4 +348,108 @@ describe('Subscriptions API', () => { await getRepository(Subscription).remove(subscription) }) }) + + describe('Subscribe API', () => { + const query = ` + mutation Subscribe($input: SubscribeInput!){ + subscribe(input: $input) { + ... on SubscribeSuccess { + subscriptions { + id + } + } + ... on SubscribeError { + errorCodes + } + } + } + ` + + context('when subscribing to a rss feed', () => { + const url = 'https://www.omnivore.app/rss' + const subscriptionType = SubscriptionType.Rss + + before(async () => { + // fake rss parser + sinon.replace(Parser.prototype, 'parseURL', sinon.fake.resolves({ + title: 'RSS Feed', + description: 'RSS Feed Description', + })) + }) + + after(() => { + sinon.restore() + }) + + context('when the user is subscribed to the feed', () => { + let existingSubscription: Subscription + + before(async () => { + existingSubscription = await createSubscription( + user.id, + 'RSS Feed', + undefined, + SubscriptionStatus.Active, + url, + subscriptionType, + url + ) + }) + + after(async () => { + await deleteSubscription(existingSubscription.id) + }) + + it('returns an error', async () => { + const res = await graphqlRequest(query, authToken, { + input: { url, subscriptionType }, + }).expect(200) + expect(res.body.data.subscribe.errorCodes).to.eql([ + 'ALREADY_SUBSCRIBED', + ]) + }) + }) + + context('when the user unsubscribed the feed', () => { + let existingSubscription: Subscription + + before(async () => { + existingSubscription = await createSubscription( + user.id, + 'RSS Feed', + undefined, + SubscriptionStatus.Unsubscribed, + url, + subscriptionType, + url + ) + }) + + after(async () => { + await deleteSubscription(existingSubscription.id) + }) + + it('re-subscribes the user', async () => { + const res = await graphqlRequest(query, authToken, { + input: { url, subscriptionType }, + }).expect(200) + expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) + expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') + }) + }) + + it('creates a rss subscription', async () => { + const res = await graphqlRequest( + query, + authToken, + { input: { url, subscriptionType } }, + ).expect(200) + expect(res.body.data.subscribe.subscriptions).to.have.lengthOf(1) + expect(res.body.data.subscribe.subscriptions[0].id).to.be.a('string') + + // clean up + await deleteSubscription(res.body.data.subscribe.subscriptions[0].id) + }) + }) + }) }) diff --git a/packages/api/test/resolvers/user.test.ts b/packages/api/test/resolvers/user.test.ts index 404f4a09d..82e0ceefc 100644 --- a/packages/api/test/resolvers/user.test.ts +++ b/packages/api/test/resolvers/user.test.ts @@ -9,7 +9,7 @@ import { findProfile } from '../../src/services/profile' import { deleteUser, findUser } from '../../src/services/user' import { hashPassword } from '../../src/utils/auth' import { createTestUser } from '../db' -import { graphqlRequest, request } from '../util' +import { generateFakeUuid, graphqlRequest, request } from '../util' describe('User API', () => { const correctPassword = 'fakePassword' @@ -238,4 +238,58 @@ describe('User API', () => { return graphqlRequest(query, invalidAuthToken).expect(500) }) }) + + describe('Delete account', () => { + const query = (userId: string) => ` + mutation { + deleteAccount( + userID: "${userId}" + ) { + ... on DeleteAccountSuccess { + userID + } + ... on DeleteAccountError { + errorCodes + } + } + } + ` + + let userId: string + let authToken: string + + before(async () => { + const user = await createTestUser('to_delete_user') + const res = await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + userId = user.id + authToken = res.body.authToken + }) + + after(async () => { + await deleteUser(userId) + }) + + context('when user id is valid', () => { + it('deletes user and responds with 200', async () => { + const response = await graphqlRequest(query(userId), authToken).expect( + 200 + ) + expect(response.body.data.deleteAccount.userID).to.eql(userId) + }) + }) + + context('when user not found', () => { + it('responds with error code UserNotFound', async () => { + const response = await graphqlRequest( + query(generateFakeUuid()), + authToken + ).expect(200) + expect(response.body.data.deleteAccount.errorCodes).to.eql([ + 'USER_NOT_FOUND', + ]) + }) + }) + }) }) diff --git a/packages/api/test/routers/rss_feed.test.ts b/packages/api/test/routers/rss_feed.test.ts new file mode 100644 index 000000000..a8e35a345 --- /dev/null +++ b/packages/api/test/routers/rss_feed.test.ts @@ -0,0 +1,101 @@ +import chai, { expect } from 'chai' +import 'mocha' +import sinon from 'sinon' +import sinonChai from 'sinon-chai' +import { User } from '../../src/entity/user' +import { SubscriptionType } from '../../src/generated/graphql' +import { createRssSubscriptions } from '../../src/services/subscriptions' +import { deleteUser } from '../../src/services/user' +import * as createTask from '../../src/utils/createTask' +import { createTestUser } from '../db' +import { request } from '../util' + +chai.use(sinonChai) + +describe('Rss feeds Router', () => { + const token = process.env.PUBSUB_VERIFICATION_TOKEN || '' + + let user: User + let user1: User + let user2: User + + before(async () => { + // create test user and login + user = await createTestUser('fakeUser') + await request + .post('/local/debug/fake-user-login') + .send({ fakeEmail: user.email }) + + user1 = await createTestUser('fakeUser1') + user2 = await createTestUser('fakeUser2') + // create test subscriptions + const name1 = 'NPR' + const url1 = 'https://www.npr.org/rss/rss.php?id=1001' + const name2 = 'BBC' + const url2 = 'http://feeds.bbci.co.uk/news/rss.xml' + await createRssSubscriptions([ + { + name: name1, + user: { id: user1.id }, + scheduledAt: new Date(), + url: url1, + type: SubscriptionType.Rss, + }, + { + name: name1, + user: { id: user2.id }, + scheduledAt: new Date(), + url: url1, + type: SubscriptionType.Rss, + }, + { + name: name2, + user: { id: user1.id }, + url: url2, + type: SubscriptionType.Rss, + }, + { + name: name2, + user: { id: user2.id }, + // 1 hour in the future + scheduledAt: new Date(Date.now() + 60 * 60 * 1000), + url: url2, + type: SubscriptionType.Rss, + }, + ]) + }) + + after(async () => { + // clean up + await deleteUser(user.id) + await deleteUser(user1.id) + await deleteUser(user2.id) + }) + + it('fetches all scheduled RSS feeds', async () => { + const data = { + message: { + data: Buffer.from('').toString('base64'), + publishTime: new Date().toISOString(), + }, + } + + // fake enqueueRssFeedFetch function + const fake = sinon.replace( + createTask, + 'enqueueRssFeedFetch', + sinon.fake.resolves('task name') + ) + + const res = await request + .post('/svc/pubsub/rss-feed/fetchAll?token=' + token) + .send(data) + .expect(200) + expect(res.text).to.eql('OK') + + // check if enqueueRssFeedFetch is called + expect(fake).to.have.been.called + + sinon.restore() + }) +}) diff --git a/packages/api/test/util.ts b/packages/api/test/util.ts index 02bac0f99..8268c1503 100644 --- a/packages/api/test/util.ts +++ b/packages/api/test/util.ts @@ -17,15 +17,14 @@ export const stopApolloServer = async () => { export const graphqlRequest = ( query: string, - authToken?: string + authToken: string, + variables?: Record, ): supertest.Test => { return request .post(apollo.graphqlPath) - .send({ - query, - }) + .send({ query, variables }) .set('Accept', 'application/json') - .set('authorization', authToken || '') + .set('authorization', authToken) .expect('Content-Type', /json/) } diff --git a/packages/content-fetch/README.md b/packages/content-fetch/README.md index 501c004ee..c72e5720f 100644 --- a/packages/content-fetch/README.md +++ b/packages/content-fetch/README.md @@ -1,6 +1,6 @@ # Puppeteer parsing function handler -This workspace is used to provide the GCF for the app to hande requests for the article parsing via Puppeteer. +This workspace is used to provide the GCF for the app to handle requests for the article parsing via Puppeteer. ## Using locally @@ -8,7 +8,7 @@ Copy .env.example file to .env file: `cp .env.example .env` Run `yarn start` to start the Google Cloud Function locally (Works without hot reloading). -After this, you should be able to access the functon on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) +After this, you should be able to access the function on [http://localhost:8080/puppeteer](http://localhost:8080/puppeteer) ## Deployment diff --git a/packages/db/migrations/0135.do.alter_user_status_type.sql b/packages/db/migrations/0135.do.alter_user_status_type.sql new file mode 100755 index 000000000..da4f39eef --- /dev/null +++ b/packages/db/migrations/0135.do.alter_user_status_type.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: alter_user_status_type +-- Description: Add DELETED to the user_status_type enum + +BEGIN; + +ALTER TYPE user_status_type ADD VALUE 'DELETED'; + +COMMIT; diff --git a/packages/db/migrations/0135.undo.alter_user_status_type.sql b/packages/db/migrations/0135.undo.alter_user_status_type.sql new file mode 100755 index 000000000..a39f1f7ba --- /dev/null +++ b/packages/db/migrations/0135.undo.alter_user_status_type.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: alter_user_status_type +-- Description: Add DELETED to the user_status_type enum + +BEGIN; + +ALTER TYPE user_status_type DROP VALUE IF EXISTS 'DELETED'; + +COMMIT; diff --git a/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql new file mode 100755 index 000000000..2bfea4029 --- /dev/null +++ b/packages/db/migrations/0136.do.add_unique_to_subscription_url.sql @@ -0,0 +1,18 @@ +-- Type: DO +-- Name: add_unique_to_subscription_url +-- Description: Add unique constraint to the url field on the omnivore.subscription table + +BEGIN; + +-- Deleting duplicates first to avoid unique constraint violation +WITH DuplicateCTE AS ( + SELECT id, ROW_NUMBER() OVER (PARTITION BY user_id, url ORDER BY status, last_fetched_at DESC NULLS LAST) AS row_number + FROM omnivore.subscriptions + WHERE type = 'RSS' +) +DELETE FROM omnivore.subscriptions + WHERE id IN (SELECT id FROM DuplicateCTE WHERE row_number > 1); + +ALTER TABLE omnivore.subscriptions ADD CONSTRAINT subscriptions_user_id_url_key UNIQUE (user_id, url); + +COMMIT; diff --git a/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql b/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql new file mode 100755 index 000000000..72071eab2 --- /dev/null +++ b/packages/db/migrations/0136.undo.add_unique_to_subscription_url.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_unique_to_subscription_url +-- Description: Add unique constraint to the url field on the omnivore.subscription table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP CONSTRAINT IF EXISTS subscriptions_user_id_url_key; + +COMMIT; diff --git a/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql b/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql new file mode 100755 index 000000000..aa7d266d8 --- /dev/null +++ b/packages/db/migrations/0137.do.alter_library_item_tsv_update_trigger.sql @@ -0,0 +1,15 @@ +-- Type: DO +-- Name: alter_library_item_tsv_update_trigger +-- Description: Alter library_item_tsv_update trigger on omnivore.library_item table to add conditions + +BEGIN; + +DROP TRIGGER IF EXISTS library_item_tsv_update ON omnivore.library_item; + +CREATE TRIGGER library_item_tsv_update + BEFORE INSERT OR UPDATE OF readable_content, site_name, title, author, description, note, highlight_annotations + ON omnivore.library_item + FOR EACH ROW + EXECUTE PROCEDURE update_library_item_tsv(); + +COMMIT; diff --git a/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql b/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql new file mode 100755 index 000000000..cb367e944 --- /dev/null +++ b/packages/db/migrations/0137.undo.alter_library_item_tsv_update_trigger.sql @@ -0,0 +1,15 @@ +-- Type: UNDO +-- Name: alter_library_item_tsv_update_trigger +-- Description: Alter library_item_tsv_update trigger on omnivore.library_item table to add conditions + +BEGIN; + +DROP TRIGGER IF EXISTS library_item_tsv_update ON omnivore.library_item; + +CREATE TRIGGER library_item_tsv_update + BEFORE INSERT OR UPDATE + ON omnivore.library_item + FOR EACH ROW + EXECUTE PROCEDURE update_library_item_tsv(); + +COMMIT; diff --git a/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql new file mode 100755 index 000000000..dfc37edfe --- /dev/null +++ b/packages/db/migrations/0138.do.add_checksum_to_subscriptions_table.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: add_checksum_to_subscriptions_table +-- Description: Add a last fetched checksum field to the subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions ADD COLUMN last_fetched_checksum TEXT ; + +COMMIT; diff --git a/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql b/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql new file mode 100755 index 000000000..e138d501a --- /dev/null +++ b/packages/db/migrations/0138.undo.add_checksum_to_subscriptions_table.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_checksum_to_subscriptions_table +-- Description: Add a last fetched checksum field to the subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP COLUMN last_fetched_checksum ; + +COMMIT; diff --git a/packages/db/migrations/0139.do.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0139.do.add_scheduled_at_to_subscription.sql new file mode 100755 index 000000000..c45201a06 --- /dev/null +++ b/packages/db/migrations/0139.do.add_scheduled_at_to_subscription.sql @@ -0,0 +1,9 @@ +-- Type: DO +-- Name: add_scheduled_at_to_subscription +-- Description: Add scheduled_at field to omnivore.subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions ADD COLUMN scheduled_at timestamptz; + +COMMIT; diff --git a/packages/db/migrations/0139.undo.add_scheduled_at_to_subscription.sql b/packages/db/migrations/0139.undo.add_scheduled_at_to_subscription.sql new file mode 100755 index 000000000..090f63ac4 --- /dev/null +++ b/packages/db/migrations/0139.undo.add_scheduled_at_to_subscription.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: add_scheduled_at_to_subscription +-- Description: Add scheduled_at field to omnivore.subscriptions table + +BEGIN; + +ALTER TABLE omnivore.subscriptions DROP COLUMN IF EXISTS scheduled_at; + +COMMIT; diff --git a/packages/db/migrations/0140.do.popular_read.sql b/packages/db/migrations/0140.do.popular_read.sql new file mode 100755 index 000000000..1c769c652 --- /dev/null +++ b/packages/db/migrations/0140.do.popular_read.sql @@ -0,0 +1,9931 @@ +-- Type: DO +-- Name: popular_read +-- Description: Create omnivore.popular_read table + +BEGIN; + +CREATE TABLE omnivore.popular_read ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), + key text NOT NULL, + original_url text NOT NULL, + title text NOT NULL, + author text NOT NULL, + description text NOT NULL, + thumbnail text, + published_at timestamptz, + site_name text NOT NULL, + slug text NOT NULL, + readable_content text NOT NULL, + original_content text NOT NULL, + word_count integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + UNIQUE (key) +); + +CREATE TRIGGER update_popular_read_modtime BEFORE UPDATE ON omnivore.popular_read FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + +GRANT SELECT ON omnivore.popular_read TO omnivore_user; + +INSERT INTO omnivore.popular_read (key, original_url, title, author, description, thumbnail, published_at, site_name, slug, readable_content, original_content, word_count) +VALUES + ('omnivore_get_started', 'https://blog.omnivore.app/p/getting-started-with-omnivore', 'Getting Started with Omnivore', 'The Omnivore Team', 'Get the most out of Omnivore by learning how to use it.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sxQnqya1QNApB7ZAGPj9K20AU6sw0UAnjmAIy2ub8hUU/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fbucketeer-e05bbc84-baa3-437e-9518-adb32be77984.s3.amazonaws.com%2Fpublic%2Fimages%2F658efff4-341a-4720-8cf6-9b2bdbedfaa7_800x668.gif', '2021-10-13', 'Omnivore Blog', 'getting-started-with-omnivore', '
      +
      +
      +

      Omnivore is a read-it-later app that lets you save and organize everything you read online.

      +
      +
      +

      This guide will show you how to use Omnivore’s basic functions and advanced features, divided into four main activities:

      +
        +
      • +

        Saving

        +
        • +
      • +

        Reading

        +
        • +
      • +

        Organizing

        +
        • +
      • +

        Integrations

        +
        • +
      +

      + The Library is the center of your Omnivore experience, where you can quickly access any links you have saved. Saved links remain in your Library forever unless you delete them. +

      +

      There are five ways to save links to pages or articles that you wish to read later:

      +
        +
      • +

        Saving from Your Omnivore Library

        +
        • +
      • +

        Saving from a Browser 

        +
        • +
      • +

        Saving from a Phone or Tablet (iOS or Android)

        +
        • +
      • +

        Newsletter Subscriptions via Email

        +
        • +
      • +

        + Saving PDFs from a Mac
        +

        +
        • +
      +

      Saving from Your Omnivore Library

      +

      + 1. In the upper right corner of your Library, tap the Add Link button.
      + 2. Enter the URL you wish to save and tap Add Link.
      + 3. The link will appear in your Library the next time you refresh it.
      +

      +

      Saving from a Browser

      +

      1. Download and install the Omnivore extension for your browser:

      + +

      + 2. Navigate to the page you wish to save and tap the Omnivore button in your browser’s toolbar or Extensions menu.
      + 3. Alternatively, you can right-click (command+click on Mac) on any hyperlink and select Save to Omnivore from the menu.
      + 4. The link will appear in your Library the next time you refresh it.
      +

      +

      Saving from a Phone or Tablet

      +

      The best way to save links from your mobile device is via the Omnivore app. You can download the app here:

      + +

      Once the mobile app is installed:

      +
        +
      1. +

        + In your browser, navigate to the page you wish to save and tap the Share button. +

        +
        2. +
      2. +

        + Tap the Omnivore icon in the Share menu. +

        +
        3. +
      3. +

        The link will appear in your Library the next time you refresh it.

        +
        4. +
      +

      Newsletter Subscriptions via Email

      +

      + 1. On the Omnivore website or app, tap your photo, initial, or avatar in the top right corner to access the profile menu. Select Emails from the menu. +

      +

      + 2. Tap Create a New Email Address to add a new email address (ex: username-123abc@inbox.omnivore.app) to the list. +

      +

      3. Click the Copy icon next to the email address.

      +

      + 4. Navigate to the signup page for the newsletter you wish to subscribe to.
      + 5. Paste the Omnivore email address into the signup form. +

      +

      6. New newsletters will be automatically delivered to your Omnivore inbox.

      +

      NOTE: If Omnivore receive's an email that does not look like an article, such as a welcome message, or note from the author, it will be forwarded to your Omnivore account email address (the email you registered with).

      +

      Saving PDFs from a Mac 

      +
        +
      1. +

        + Install the Mac App.  +

        +
        2. +
      2. +

        On your Mac, locate the PDF you wish to save and right-click or ctrl+click on the file name.

        +
        3. +
      3. +

        + Select Share from the menu and choose Omnivore. +

        +
        4. +
      4. +

        The link will appear in your Library the next time you refresh it.

        +
        5. +
      +

      Reading

      +

      Click any link saved in your Library to enter the Reader view. 

      +

      Omnivore formats pages for easy reading and highlighting, removing ads and clutter for distraction-free reading. The text-focused view also makes articles smaller and quicker to load.

      +

      While reading, you can:

      +
        +
      • +

        Change Formatting

        +
        • +
      • +

        Highlight Text

        +
        • +
      • +

        Add Notes

        +
        • +
      • +

        View All Saved Highlights and Notes

        +
        • +
      • +

        Track Reading Progress

        +
        • +
      +

      Change Formatting 

      +
        +
      1. +

        + Theme: Tap your photo, initial, or avatar  in the top right corner to access the profile menu. Select the white or black thumbnail to choose the Light or Dark theme. +

        +
        2. +
      2. +

        + Text Formatting: Tap the Aa icon to adjust the text size, font, margins, and line spacing. +

        +
        3. +
      +

      Highlight Text

      +
        +
      1. +

        Select the text you wish to highlight.

        +
        2. +
      2. +

        + Tap the Highlight button. +

        +
        3. +
      3. +

        The text will appear highlighted next time you view the article.

        +
        4. +
      +

      Add Notes

      +
        +
      1. +

        Highlight a section of text where you wish to add a note.

        +
        2. +
      2. +

        + Tap the Note button, type your note, and tap Save. +

        +
        3. +
      3. +

        The Note icon will appear next time you view this article.

        +
        4. +
      +

      View All Saved Highlights and Notes

      +
        +
      1. +

        Tap the Highlight/Note icon to see a list of all the highlighted text and notes you have added to this page.

        +
        2. +
      2. +

        To remove a note or highlight, select it from the list and tap the Trash icon.

        +
        3. +
      +

      Track Reading Progress

      +

      Omnivore automatically keeps track of your reading progress across your different devices so you can easily pick up where you left off. A progress bar will appear at the top of each link in your Library after you have started reading.

      +

      Organizing

      +

      By default, the Library inbox displays all links you have saved. To manage your list and keep your reading organized, Omnivore provides the following actions: 

      +
        +
      • +

        Archiving

        +
        • +
      • +

        Labels

        +
        • +
      • +

        Search

        +
        • +
      • +

        Filters

        +
        • +
      +

      Archiving

      +
        +
      1. +

        Tap the Menu icon next to the link you wish to archive (on the mobile app, long press the link to open the menu).

        +
        2. +
      2. +

        + Select Archive. +

        +
        3. +
      3. +

        The link will disappear from the default Library view, but will show up if you select the Archived filter (see Filters below).

        +
        4. +
      +

      + Labels +

      +
        +
      1. +

        + Tap the Menu icon next to any link and select Set Labels. +

        +
        2. +
      2. +

        + Select an existing label from the list or tap Edit Labels to create a new one. +

        +
        3. +
      3. +

        The label will appear next to the link in your Library. Tap it to view all links with the same label.

        +
        4. +
      4. +

        + Omnivore mobile app only: tap Labels to see a complete list of all labels you have used; tap one to view all links with the same label +

        +
        5. +
      5. +

        Note: Omnivore will automatically assign some labels, such as “Newsletters.”

        +
        6. +
      +

      Search

      +
        +
      1. +

        To search through all your saved links, enter a keyword or phrase in the search bar. 

        +
        2. +
      2. +

        + You can combine keywords with labels and filters to focus your search even further. Learn more about advanced search. +

        +
        3. +
      +

      Filters

      +
        +
      1. +

        + Use the Filters menu to refine your Library view (some filters may be visible by default). +

        +
        2. +
      2. +

        + Select Read Later to view a list of all your non-archived links except Newsletters. +

        +
        3. +
      3. +

        + Select Highlights to view the text selections you have highlighted in all your saved pages.  +

        +
        4. +
      4. +

        + Select Today to view a list of links you saved today. +

        +
        5. +
      5. +

        + Select Newsletters to view links saved via your newsletter subscriptions. +

        +
        6. +
      +

      Integrations

      +

      Omnivore allows integrations with knowledge bases and note-taking apps including:

      +
        +
      • +

        Logseq

        +
        • +
      • +

        Webhooks

        +
        • +
      +

      Logseq

      +

      + With Omnivore''s Logseq plugin you can sync all your saved articles, highlights, and notes into Logseq, a popular knowledge base. For information on setting up and using the Logseq plugin, please refer to this helpful Omnivore for Logseq Plugin Guide. +

      +

      Webhooks

      +

      + Omnivore can trigger webhooks when you save a link or add highlights to a page you are reading. This example shows webhooks being used to write all saved links to a Google Sheets spreadsheet stored on a Google Drive. +

      +
      +
      +
      +', ' + + + + + + + + + + + + + + + + + + + + + + + + Getting Started with Omnivore - Omnivore + + + + + + + + + + + + + + + + + + + + +
      +
      + +
      +
      +
      +
      +
      + +
      +

      + Omnivore +

      +
      + +
      +
      +
      +
      +
      +
      +
      + + + + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      + TopNew + + + + + +
      +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore +
      +
      +
      +
      + Omnivore +
      +
      +
      + +
      +
      + + + + +
      + 2 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore +
      +
      +
      +
      + Omnivore +
      +
      +
      + +
      +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Add to your library with your Omnivore email address +
      +
      +
      +
      + Omnivore +
      +
      +
      + +
      +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      See all + + + + +
      +
      +
      +
      +
      + +
      + +
      +
      +
      +
      +
      +
      + © 2022 Omnivore +
      +
      + Privacy ∙ Terms ∙ Collection notice +
      +
      +
      + + + + + + + Publish on Substack + + + + + Get the app +
      +
      + Substack is the home for great writing +
      +
      +
      +
      +
      +
      +
      + + + + + + + +', 1155), + ('omnivore_ios', 'https://blog.omnivore.app/p/saving-links-from-your-iphone-or', 'Saving Links from Your iPhone or iPad', 'Omnivore', 'Learn how to save articles on iOS.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sWDfv7sARTIdAlx6Rw_6t-QwL3T9aniEJRa1-jVaglNg/https://substackcdn.com/image/youtube/w_728,c_limit/k6RkIqepAig', '2021-10-19', 'Omnivore Blog', 'saving-links-from-your-i-phone-or-i-pad', '
      +
      +
      +

      + With the Omnivore app for iOS, it’s easy to save web pages and articles or archive web content to read later. +

      +

      + The Omnivore app uses the iOS Share System, which lets you send items from one app (such as Safari) to another (such as Messages or Mail).  +

      +
        +
      • +

        Step 1: Log in to the Omnivore app.

        +
        • +
      • +

        Step 2: Add Omnivore to your Share menu favorites.

        +
        • +
      • +

        Step 3: Save links to your Omnivore Library.

        +
        • +
      +
      +

      + +

      +
      +

      You must be logged in before you can save links via the Share menu. If you don’t already have an Omnivore account, you can sign up for free from the login screen.

      +

      + Note: If you haven’t installed the iOS app, download it here: https://omnivore.app/install/ios +

      +

      + Step 2: Add Omnivore to your Share menu favorites. +

      +

      Start by viewing the Share menu from within any supported iOS app (we’ve used Safari for this example).

      +
        +
      1. +

        + Tap the Share icon at the bottom of the screen. +

        +
        2. +
      2. +

        + Swipe left to the end of the list of app icons and tap More. +

        +
        3. +
      3. +

        + Tap Edit at the top of the screen. +

        +
        4. +
      4. +

        + Scroll down until you see the Omnivore icon and tap the + icon next to it.  +

        +
        5. +
      5. +

        + Press and hold the three-bar icon and drag Omnivore to one of the top positions under Favorites. Tap Done to close the menu. +

        +
        6. +
      6. +

        + Omnivore will appear as one of the first options the next time you use the Share feature (you may need to restart Safari). +

        +
        7. +
      +

      + Step 3: Save links to your Omnivore Library +

      +

      + Start by navigating to the page or article you wish to save. Please note that Omnivore will save the content that appears on your screen (not just a link), so if the page is behind a paywall and you are logged into the paywalled site, you will save the paid content.  +

      +
        +
      1. +

        + While viewing the page you’d like to save, tap the Share icon. +

        +
        2. +
      2. +

        + Tap the Omnivore icon in the Share menu. +

        +
        3. +
      3. +

        + Tag the article with one or more labels (optional) and tap Read Now or Read Later.  +

        +
        4. +
      4. +

        If you choose Read Later, the link will appear in your Library the next time you open the Omnivore app.

        +
        5. +
      +
      +
      +
      ', ' + + + + + + + + + + + + + + + + + + + + + + + + Saving Links from Your iPhone or iPad - Omnivore + + + + + + + + + + + + + + + + + + + + + +
      +
      + +
      +
      +
      +
      +
      + +
      +

      + Omnivore +

      +
      + +
      +
      +
      +
      +
      +
      +
      + + + + +
      +
      +
      + +
      +
      +
      +
      +
      +
      + Comments +
      +
      +
      +
      +
      +
      + +
      + +
      +
      +
      +
      + +
      +
      +
      +
      + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      + TopNew + + + + + +
      +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore +
      +
      +
      +
      + Omnivore +
      +
      +
      + +
      +
      + + + + +
      + 2 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore +
      +
      +
      +
      + Omnivore +
      +
      +
      + +
      +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      +
      +
      + +
      +
      + Add to your library with your Omnivore email address +
      +
      +
      +
      + Omnivore +
      +
      +
      + +
      +
      + + + + +
      + 1 +
      + + + + + +
      +
      +
      See all + + + + +
      +
      +
      +
      +
      + +
      + +
      +
      +
      +
      +
      +
      + © 2022 Omnivore +
      +
      + Privacy ∙ Terms ∙ Collection notice +
      +
      +
      + + + + + + + Publish on Substack + + + + + Get the app +
      +
      + Substack is the home for great writing +
      +
      +
      +
      +
      +
      +
      + + + + + + + +', 371), + ('omnivore_organize', 'https://blog.omnivore.app/p/organize-your-omnivore-library-with', 'Organize your Omnivore library with labels', 'The Omnivore Team', 'Use labels to organize your Omnivore library.', 'https://proxy-prod.omnivore-image-cache.app/320x320,sTgJ5Q0XIg_EHdmPWcxtXFmkjn8T6hkJt7S9ziClagYo/https://substackcdn.com/image/fetch/w_1200,h_600,c_fill,f_jpg,q_auto:good,fl_progressive:steep,g_auto/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fdaf07af7-5cdb-4ecc-aace-1a46de3e9c58_1827x1090.png', '2022-04-18', 'Omnivore Blog', 'organize-your-omnivore-library-with-labels', '
      +
      +

      Omnivore provides labels (also known as tags) to help you organize your library. Labels can be added to any saved read, and your library can be filtered based on labels.

      +

      On the web if you have a larger screen you can find the labels tool on the left side of the screen.

      +
      +
      + + + + + +
      Adding a label from the left menu
      +
      +
      +

      For a smaller screen you will find the labels button at the top of the page.

      +
      +
      + + + + + +
      Article Actions at the top of the Omnivore Reader for smaller screens
      +
      +
      +

      iOS users can long press on an item in the library or access the labels modal from the menu in the top right of the reader page.

      +
      +
      + + + + + +
      Editing labels from the library view on iOS
      +
      +
      +

      Label searches on iOS

      +

      On iOS you can use the label search modal to search for specific labels. This will create an OR search and return all links matching the assigned labels.

      +
      +
      + + + + + +
      Using labels to filter your search
      +
      +
      +

      Using Advanced Search to filter your library with labels

      +

      Omnivore''s advanced search syntax supports searching for multiple labels using AND and OR clauses. You can also negate a label search to find pages that do not have a certain label.

      +

      Some examples:

      +
        +
      • +

        + label:Newsletter finds all pages that have the label Newsletter +

        +
        • +
      • +

        + label:Cooking,Fitness finds all your pages with either the Cooking or Fitness labels +

        +
        • +
      • +

        + label:Newsletter label:Surfing finds all pages with both the Newsletter and Surfing labels +

        +
        • +
      • +

        label:Coding -label:News finds all pages with the Coding label that do not have the News label

        +
        • +
      +
      +
      +', ' + + + + + + + + + + + + + + + + + + + + + + + + Organize your Omnivore library with labels - Omnivore + + + + + + + + + + + + + + + + + +
      +
      + +
      +
      +
      +
      +
      + +
      +

      + Omnivore +

      +
      + +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      + TopNew + + + + + +
      +
      +
      +
      + +
      +
      + Learn the best ways to save links with Omnivore +
      +
      + +
      +
      + +
      +
      + +
      +
      + + + + Share + + + + + + +
      +
      +
      +
      +
      +
      + +
      +
      + Highlighted <code> in Omnivore +
      +
      + +
      +
      + +
      +
      + +
      +
      + + + + Share + + + + + + +
      +
      +
      +
      +
      +
      + +
      +
      + Today we are happy to launch our new PDF viewer. It is available in our latest iOS release (1.3.0) and on the web. The new PDF viewer supports… +
      +
      + +
      +
      + +
      +
      + +
      +
      + + + + Share + + + + + + +
      +
      +
      +
      See all + + + + +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      + © 2022 Omnivore +
      +
      + Privacy ∙ Terms ∙ Collection notice +
      +
      +
      + + + + + + + Publish on Substack + + + + + Get the app +
      +
      + Substack is the home for great writing +
      +
      +
      +
      +
      +
      + + + + + + + + +', 259), + ('rlove_carnitas', 'https://medium.com/@rlove/carnitas-ff0ef1044ae9', 'Slow-Braised Carnitas Recipe', 'Robert Love', 'Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. This is not an authentic recipe.', 'https://proxy-prod.omnivore-image-cache.app/88x88,sIcDXt3Ar0baKG1e1Yi1e2VUZFL85xPlOeEfAxF-s-Nw/https://miro.medium.com/max/1200/1*Wl-dMBJpSgPUxUOnPQthyg.jpeg', '2017-02-24', '@rlove', 'slow-braised-carnitas-recipe', '
      +
      +
      +
      +

      I used to have a bunch of recipes up online. But writing recipes is no fun; it is difficult to capture the beauty of a dish with a bunch of steps. Moreover, using recipes isn’t how I cook. I want to understand the flavors of a dish and then execute it in my own way, in my own hands. So the recipes went away.

      +

      But one of the most popular — and one of my personal favorites — was a recipe for the Mexican pork dish carnitas. It was a fun, relatively easy recipe, not traditional in approach but fairly traditional (and really delicious) in output. Folks keep asking for it. So here it is, in hopes I can eat it at your next house party.

      +

      Carnitas is a wonderful Mexican dish, pork shoulder cooked until tender and then given a great crisp. In Mexico, carnitas is eaten on its own, in tacos, or in tortas. The traditional recipe is simple: several pounds of pork shoulder, a pound or two of lard, orange peel, and some water (or coca-cola), slow roasted and then “boiled” to a crisp. That is…a bit much. What follows is not an authentic approach.

      +
      +

      +

      +
      Photo by flickr user mccun934 licensed CC BY 2.0 +
      +
      +

      Never ones to eschew taste for health, the French have nonetheless taught us nothing if not that we can make a succulent, flavorful dish without boiling a tough cut of meat in lard. In that vein, my recipe is more of a braiser de porc than a confit de porc — pork shoulder, aromatics, citrus juice, and a little Grand Marnier, slow cooked on the stove. Healthier for the heart, but also — more importantly — tastier to the tongue. To obtain that classic carnitas crisp, we braise the meat uncovered until the liquid evaporates and then move the pot to the oven and caramelize.

      +

      Makes about 6 servings.

      +

      Ingredients:

      +

      3 lbs boneless pork shoulder, preferably Boston butt (the upper shoulder)
      4 tablespoons grape seed oil
      6 cloves garlic, peeled and thinly-sliced
      2 oranges, juiced, plus the zest of 1/2
      2 limes, juiced and zested
      several cups chicken stock
      1 teaspoon ground cayenne pepper
      1 tablespoon ground ancho chile
      1 tablespoon ground chipotle pepper
      2 teaspoons cumin
      1 teaspoon ground cinnamon
      1 teaspoon freshly-ground black pepper
      + herb sachet with 8 sprigs Mexican oregano, 6 sprigs thyme, 2 sticks cinnamon, and 2 bay leaves, wrapped in a cheese cloth and tied shut with cooking twine
      1 cup Grand Marnier
      coarse sea salt, to coat meat, plus more to taste +

      +

      Cut the pork shoulder into 5" chunks. Remove any gratuitously-excessive fat, but leave at least a thin layer. Sprinkle the chunks with sea salt. Let sit at room temperature at least 30 minutes. Pat dry.

      +

      Heat the grape seed oil in a dutch oven over medium-high heat. Sauté the pork shoulder until well browned on each side. If needed, sauté across multiple batches.

      +

      Add the garlic and sauté for one more minute.

      +

      Add the orange zest, lime zest, ground cayenne pepper, ground ancho chile, ground chipotle chile, cumin, ground cinnamon, black pepper, and herb sachet. Mix.

      +

      Add the Grand Marnier, orange juice, and lime juice. Stir, scrapping the bottom of the pot.

      +

      Add chicken stock as needed such that the pork is two-thirds submerged in liquid.

      +

      Stirring, raise heat to high and bring to a boil. Once boiling, lower heat until the liquid is at a simmer and braise, uncovered, stirring occasionally, until the pork is cooked and tender but not disintegrating and the liquid is reduced by at least two-thirds, about three hours. If the liquid gets perilously-low while cooking, add a little chicken stock.

      +

      Remove pork from pot. Once cool enough to handle, use a fork to shred the pork into bite-sized, but fairly large, chunks, removing any fatty pieces as desired.

      +

      Preheat oven to 450°F.

      +

      Return pork chunks to pot. Place uncovered pot in oven. Continue cooking until the liquid has evaporated and the pork is crispy and starting to caramelize, about 20 minutes.

      +

      Taste and adjust salt. Serve with warm corn tortillas, guacamole, pico de gallo, diced white onion, chopped cilantro, lime wedges, margaritas, and college football.

      +
      +
      +
      +
      ', ' + + + + Slow-Braised Carnitas Recipe. Robert Love’s recipe for carnitas, in… | by Robert Love | Medium + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      +
      + +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      + Robert Love +
      +
      +
      +
      +
      +
      +
      + +
      +
      + +
      +
      +
      +
      +

      + Feb 24, 2017 +

      + +
      + 3 min read +
      + +
      +
      +
      +
      +
      + +

      + +

      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +

      + +

      +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + Slow-Braised Carnitas Recipe +

      +
      + + + +
      +
      + +
      +
      + Photo by flickr user mccun934 licensed CC BY 2.0 +
      +
      + + + + + + + + + + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      + + +
      +
      +
      +
      + +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      + + +
      +
      +
      +
      + +
      +
      +
      +
      +
      + +
      + + +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + More from Robert Love +

      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +

      + Google, Android, Linux kernel, author, Boston +

      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + Love podcasts or audiobooks? Learn on the go with our new app. +

      +
      +
      + Try Knowable +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + + +
      +

      + AboutHelpTermsPrivacy +

      +
      +
      + +

      + Get the Medium app +

      +
      +
      + A button that says ''Download on the App Store'', and if clicked it will lead you to the iOS App store +
      A button that says ''Get it on, Google Play'', and if clicked it will lead you to the Google Play store +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      + +
      +
      +
      +
      +
      + +
      +
      +
      + Robert Love +
      +
      +
      +

      + Robert Love +

      +
      +
      +

      + Google, Android, Linux kernel, author, Boston +

      +
      +
      + +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      +

      + More from Medium +

      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + Back2Life +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +

      + This Produce Season: Winter +

      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + Jeannie @ Paw It Forward +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +

      + Eli’s Level 2 Board & Train: Small Dog with Big Dog Syndrome +

      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +
      +
      +
      + Amit Sarda +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +
      +

      + in +

      +
      +
      + +

      + Simple Cooking +

      +
      +
      +
      +

      + Masala Chai Recipe +

      +
      +
      +
      +
      +
      + A cup of masala chai (Indian spiced tea) in a green plate +
      +
      +
      +
      +
      + +
      + +
      +
      +
      +
      +
      +
      + Australian Foodie +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      +

      + in +

      +
      +
      + +

      + Global Foodies +

      +
      +
      +

      + Vegemite Spaghetti Recipe — An Easy One Pot Dinner +

      +
      +
      +
      +
      + Vegemite pasta spaghetti recipe +
      +
      +
      +
      +
      + +

      + Help +

      +
      +
      + +

      + Status +

      +
      +
      + +

      + Writers +

      +
      +
      + +

      + Blog +

      +
      +
      + +

      + Careers +

      +
      +
      + +

      + Privacy +

      +
      +
      + +

      + Terms +

      +
      +
      + +

      + About +

      +
      +
      + +

      + Knowable +

      +
      +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +', 705), + ('power_read_it_later', 'https://fortelabs.co/blog/the-secret-power-of-read-it-later-apps', 'The Secret Power of ‘Read It Later’ Apps', 'Tiago Forte', 'At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket', 'https://proxy-prod.omnivore-image-cache.app/320x320,sGN5R34M5z068QMXDZD32CQD6mCbxc47hWXm__JVUePE/https://fortelabs.com/wp-content/uploads/2015/11/1rPXwIczUJRCE54v8FfAHGw.jpeg', '2022-01-24', 'Forte Labs', 'the-secret-power-of-read-it-later-apps', '
      +
      +
      +
      +
      + +
      Image via Nuno Cruz
      +
      +
      +
      +

      + By Tiago Forte of Forte Labs +

      +

      At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year.

      +

      + +

      +

      This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask.

      +

      What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place.

      +

      We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours.

      +

      The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading.

      +

      The fact is, the ability to read is becoming a source of competitive advantage in the world.

      +

      I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time.

      +

      A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.”

      +
      +
      +

      +

      Start Building Your Second Brain

      +

      Subscribe below to learn more about the next cohort of the Building a Second Brain course

      +
      +
      +

      Read It. Later.

      +

      Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar.

      +

      So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout.

      +

      On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are:

      + +

      The app I use, Pocket, adds a button to the Chrome toolbar that looks like this:

      +
      + +
      Chrome toolbar
      +
      +

      + Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. +

      +

      Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved:

      +
      +
      + +
      Mac desktop client
      +
      +

      You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self:

      +
      +
      + +
      Web browser “tile” view
      +
      +

      Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces).

      +

      The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development

      +

      I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity.

      +

      But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out.

      +

      There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with:

      +
        +
      1. Increasing consumption of long-form content (which is presumably more substantive)
        2. +
      2. Better filtering
        3. +
      +

      #1 | Increasing Consumption of Long-Form Content

      +

      In order to consume good ideas, first you have to consume many ideas.

      +

      This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation.

      +

      The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances.

      +
      +

      Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. +

      +
      +

      Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them:

      +

      1. App performance

      +

      We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off.

      +

      If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal.

      +

      Meanwhile, your snaps and instas refresh at precog-like speeds.

      +

      Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about.

      +

      A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off.

      +
      + +
      Effect of ad-blocker on loading times of Boston.com, via NYT +
      +
      +

      Yeah that’s pretty much an eternity in mobile behavior land.

      +

      2. Matching content with your context

      +
      +
      + +
      My Pocket list on iPad
      +
      +
      +

      Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse.

      +

      So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in.

      +

      Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment.

      +

      David Allen puts it this way:

      +
      +

      “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.”

      +
      +

      You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓

      +

      3. Asynchronous reading

      +

      This is one of the least understood barriers to reading in our fragmented timescape.

      +

      There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article.

      +

      The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it.

      +

      Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished.

      +

      4. Focus

      +

      A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it.

      +

      It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing.

      +

      Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read.

      +
      +

      Actionable info ≠ Reference info ≠ To Read pile

      +
      +

      Ergo,

      +
      +

      Task manager ≠ Evernote ≠ Pocket

      +
      +

      #2 | Better filtering

      +

      Now you’ve got the funnel filled. It’s time to narrow it.

      +

      Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources.

      +

      The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape.

      +

      Yeah.

      +

      In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you.

      +

      Procrastination is the most powerful force in the universe. It will find a way. +

      +

      I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time.

      +

      I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session.

      +

      So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab.

      +

      Here’s what a typical Monday morning link-fest looks like, just from email:

      +

      + +

      +

      Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing).

      +

      There’s only one rule: NO READING OR WATCHING! +

      +

      Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in.

      +

      I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about.

      +

      What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me.

      +
      +

      Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide

      +
      +

      Here’s a visual of how this works, from my Pocket analytics:

      +

      + +

      +

      You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week.

      +

      + /sidebar +

      +

      Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back.

      +

      I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on.

      +

      If only this method would scale.

      +

      + /end_sidebar +

      +

      Mo’ apps, mo’ problems

      +

      There are drawbacks, which I’ve glossed over until now. The two main ones:

      +

      1. Formatting issues

      +

      Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker.

      +

      The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed.

      +

      One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer.

      +

      2. Dependence

      +

      Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me.

      +

      For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved.

      +

      You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow:

      +

      + Highlight > Share > Share to Evernote > Save +

      +

      …has been replaced with this:

      +

      + Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket +

      +

      Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date.

      +

      + /rant_over +

      +

      Progress Traps and Paradigms

      +

      The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little.

      +

      As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know.

      +

      As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid.

      +

      What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go?

      +

      I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.”

      +

      When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens.

      +

      Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere.

      +

      But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on.

      +

      There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself.

      +

      Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain.

      +

      I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms.

      +

      I can’t put it any better than Donella Meadows, in her seminal piece on complex systems:

      +
      +

      People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. +

      +
      +
      +

      It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia.

      +
      +
      +

      In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. +

      +
      +

      Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own.

      +

      The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds.

      +

      And that is the secret power of Read It Later apps.

      +

      + P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) +

      +
      +

      Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.

      +
      +
      +

      +

      Join the Forte Labs Newsletter

      +

      Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I''ll send you my Top 10 All-Time Articles right away as a thank you.

      +
      +
      +
      + +
      +
      +', ' + + + + + + + + + + + The Secret Power of ‘Read It Later’ Apps - Forte Labs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +

      + The Secret Power of ‘Read It Later’ Apps +

      +
      +
      +
      +
      + +
      +
      +
      +
      +
        +
      • + Last modified +
        • +
      • + January 24, 2022 +
        • +
      +
      +
      +
      +
      +
      +

      + Estimated reading time: 14 minutes +

      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      + Image via Nuno Cruz +
      +
      +
      +
      +

      + By Tiago Forte of Forte Labs +

      +

      + At the end of 2014 I received an email informing me that I had read over a million words in the ‘read it later’ app Pocket over the course of the year. +

      +

      + +

      +

      + This number by itself isn’t impressive, considering our daily intake of information is equivalent to 34 gigabytes, 100,000 words, or 174 newspapers, depending on who you ask. +

      +

      + What makes this number significant (in my view) is that it represents 22 books’-worth of long-form reading that would not have happened without a system in place. +

      +

      + We’ve made a habit of filling those hundred random spaces in our day with glances at Twitter, Instagram, and Facebook. But those glances have slowly become stares, and those stares have grown to encompass a major portion of our waking hours. +

      +

      + The end result is the same person who spends 127 hours per year on Instagram (the global average) complains that she has “no time” for reading. +

      +

      + The fact is, the ability to read is becoming a source of competitive advantage in the world. +

      +

      + I’m not talking about basic literacy. What has become exceedingly scarce (and therefore, valuable) is the physical, emotional, attentional, and mental capability to sit quietly and direct focused attention for sustained periods of time. +

      +

      + A recent article in the Harvard Business Review puts a name to this new neurological phenomenon: Attention Deficit Trait. Basically, the terms ADD and ADHD are falling out of use because effectively the entire population fits the diagnostic criteria. It’s not a condition anymore, it’s a trait — the inherent and unavoidable experience of modern life characterized by “distractibility, inner frenzy, and impatience.” +

      +
      +
      +
      +
      +
      +

      + Start Building Your Second Brain +

      +
      +
      +

      + Subscribe below to learn more about the next cohort of the Building a Second Brain course +

      +
      +
      +
      +
        +
        +
        + +
        + +
        +
        +
        +
        +

        +   +

        +

        + Read It. Later. +

        +

        + Before I explain the massive, under-appreciated benefits these apps provide, and how to use them most effectively, a quick primer in case you’re unfamiliar. +

        +

        + So-called “Read It Later” apps give you the ability to “save” content on the web for later consumption. They are essentially advanced bookmarking apps, pulling in the content from a page to be read or viewed in a cleaner, simpler visual layout. +

        +

        + On top of that core function they add features like favoriting, tags, search, cross-platform syncing, recommended content, offline viewing, and archiving. The most popular options are: +

        + +

        + The app I use, Pocket, adds a button to the Chrome toolbar that looks like this: +

        +
        + +
        + Chrome toolbar +
        +
        +

        + Note: at time of writing, I was using Pocket, but have recently switched to Instapaper because of Pocket’s “Share to Evernote” bug mentioned below. +

        +

        + Clicking the button while viewing a webpage turns the button pink, and saves the page to your “list.” Navigating to getpocket.com, or opening the Pocket app on your computer or mobile device shows you a list of everything you’ve saved: +

        +
        +
        + +
        + Mac desktop client +
        +
        +

        + You can also view your list in a “tile” layout on the web, making it into essentially a personalized magazine. Personalized, in this case, not by a cold, unfeeling algorithm, but by your past self: +

        +
        +
        + +
        + Web browser “tile” view +
        +
        +

        + Marking an item as read in one version of the app will quickly sync across all platforms. It will also save your current progress on one device, so you can continue where you left off on a different device (for those longer pieces). +

        +

        + The highest leverage point in a system is in the intake — the initial assumptions and paradigms that inform its development +

        +

        + I’ve written previously about how to use Evernote as a general reference filing system, not only to stay organized but to inspire creativity. +

        +

        + But I didn’t address a key question when creating any workflow: how and from where does information enter the system? The quality of a workflow’s outputs is fundamentally limited by the quality of its inputs. Garbage in, garbage out. +

        +

        + There are A LOT of ways we could talk about to improve the quality of the information you consume. But I want to focus now on the two that Read It Later apps can help with: +

        +
          +
        1. Increasing consumption of long-form content (which is presumably more substantive) +
          2. +
        2. Better filtering +
          3. +
        +

        + #1 | Increasing Consumption of Long-Form Content +

        +

        + In order to consume good ideas, first you have to consume many ideas. +

        +

        + This is the fundamental flaw in the “information diet” advice from Tim Ferriss and others: strong filters work best on a larger initial flow. Using your friends as your primary filter for new ideas ensures you remain the dumbest person in the room, and contribute nothing to the conversation. +

        +

        + The problem is that our entire digital world is geared toward snackable chunks of low-grade information — photos, tweets, statuses, snaps, feeds, cards, etc. To fight the tide you have to redesign your environment — you have to create affordances. +

        +
        +

        + Affordance (n.): a relation between an object and an organism that, through a collection of stimuli, affords the opportunity for that organism to perform an action. +

        +
        +

        + Let’s look at the 4 main barriers to consuming long-form content, and the affordances that Read It Later apps use to overcome them: +

        +

        + 1. App performance +

        +

        + We know that the most infinitesimal delays in the loading time of a webpage will dramatically impact how many people stay on the page. Google found that increasing the number of results per page from 10 to 30 took only half a second longer, but caused 20% of people to drop off. +

        +

        + If you think your behavior is not affected by such trivialities, think again. Even on a subconscious level, you will resist even opening apps that don’t reward you with snappy response times. Which is a problem because the apps most people turn to for reading are either ebook apps like iBooks and Kindle, or web browsers like Chrome and Safari. I’m not sure which category is slower, but they’re both abysmal. +

        +

        + Meanwhile, your snaps and instas refresh at precog-like speeds. +

        +

        + Read It Later apps, by slurping in content (articles, videos, slideshows) into a clean interface, eliminate the culprits — ads, site analytics, popups — all the stuff you don’t care about. +

        +

        + A recent analysis by The New York Times of 3 leading ad-blockers (which have the same effect) measured a 21% increase in battery life, and in the most egregious case of Boston.com, a drop in loading time from 33 seconds to 7 seconds. Many other leading sites were not that far off. +

        +
        + +
        + Effect of ad-blocker on loading times of Boston.com, via NYT +
        +
        +

        + Yeah that’s pretty much an eternity in mobile behavior land. +

        +

        + 2. Matching content with your context +

        +
        +
        + +
        + My Pocket list on iPad +
        +
        +
        +

        + Much of the time when we pull out our phone, we’re looking for something to match our mood (or energy, or time available, or other context). We use our constellation of shiny apps as mood regulators and self-soothers, as time-fillers and boredom-suppressors, for better or worse. +

        +

        + So you need a little entertainment, and you open…an ebook? Yeah right. Monochrome pages don’t attract you. They don’t draw you in. +

        +

        + Pocket gives reading some of this stimulatory pleasure by laying out your list in a pleasing, magazine-style layout (at left). Not only is it generally attractive, but it gives you that same magazine-flipping pleasure of engaging with something that interests you right in that moment. +

        +

        + David Allen puts it this way: +

        +
        +

        + “It’s practical to have organized reading material at hand when you’re on your way to a meeting that may be starting late, a seminar that may have a window of time when nothing is going on, a dentist appointment that may keep you waiting, or, of course, if you’re going to have some time on a train or plane. Those are all great opportunities to browse and work through that kind of reading. People who don’t have their Read/Review material organized can waste a lot of time, since life is full of weird little windows when it could be used.” +

        +
        +

        + You’re not fighting your impulses forcing yourself to read a dense tome after a long work day. Willpower preserved ✓ +

        +

        + 3. Asynchronous reading +

        +

        + This is one of the least understood barriers to reading in our fragmented timescape. +

        +

        + There is something deeply, deeply unsatisfying about repeatedly starting something and not finishing it. This is what we experience all day at work, being continuously interrupted by a stream of “emergencies.” The last thing we want after a stressful day starved of wins is to fail even at reading an article. +

        +

        + The 2015 revised edition (affiliate link) of Getting Things Done cites the work of Dr. Roy Baumeister, who has shown that “uncompleted tasks take up room in the mind, which then limits clarity and focus.” The risk of cognitive dissonance at not being able to finish a long article (much less a book) keep us from even beginning it. +

        +

        + Read It Later apps address this by simply saving your progress in a given article, allowing you to pick back up at a different time, or on a different device, and clearly marking items as “read” once you’re finished. +

        +

        + 4. Focus +

        +

        + A common response when I recommend people adopt yet another category of apps is “Why don’t I just use Evernote?” Or whatever app they’re using for general reference or task management. Evernote even makes a Chrome extension called Clearly for reading online content and Web Clipper for saving it. +

        +

        + It is a question of focus. Why don’t you use your task manager to keep track of content (i.e. “Read this article”)? Because the last thing you want to see when you cuddle up with your hot cocoa for some light reading is the hundreds of tasks you’re not doing. +

        +

        + Likewise, the last thing you want to see when you (finally!) have time to read is the thousands of notes you’ve collected from every corner of the universe, only some of which you haven’t read, only some of which you want to read, only some of which are meant to be read. +

        +
        +

        + Actionable info ≠ Reference info ≠ To Read pile +

        +
        +

        + Ergo, +

        +
        +

        + Task manager ≠ Evernote ≠ Pocket +

        +
        +

        + #2 | Better filtering +

        +

        + Now you’ve got the funnel filled. It’s time to narrow it. +

        +

        + Most advice on this topic focuses on being more selective about your sources. Cutting out the email digests that just throw you off track, unfollowing people posting crap, or even directly replacing ads with quality sources. +

        +

        + The problem is that this assumes you are always at your best, always at 100% self-discipline, totally aligned with your life values, priorities ship shape. +

        +

        + Yeah. +

        +

        + In the moment, with your blood sugar at a negative value and every fiber of your being screaming for a dopamine hit, of course that Buzzfeed article seems like the best conceivable use of your time. If you think you can permanently seal off your life from the celebrity news, content marketing, and spammy friends that dominate the web, the NSA has a job for you. +

        +

        + Procrastination is the most powerful force in the universe. It will find a way. +

        +

        + I have a different approach: waiting periods. Every time I come across something I may want to read/watch, I’m totally allowed to. No limits! The only requirement is I have to save it to Pocket, and then choose to consume it at a later time. +

        +

        + I’ve found that even just clicking a link to open the URL, in order to save it to Pocket, is too much of a temptation. The first glimpse of a cute GIF and I’m off to Reddit, completely forgetting my morning email session. +

        +

        + So instead I just command-click every link I’m interested in (or right-click > Open link in new tab), which opens each link in a separate tab without taking me to that tab. +

        +

        + Here’s what a typical Monday morning link-fest looks like, just from email: +

        +

        + +

        +

        + Then, because I’m still in collection mode, not in read mode, I cycle through each tab one at a time (shift-command-} or control-tab), saving each one to Pocket using the shortcut I set up: command-p (chosen for irony and to avoid inadvertent printing). +

        +

        + There’s only one rule: NO READING OR WATCHING! +

        +

        + Bringing this back to filtering, not only am I saving time and preserving focus by batch processing both the collection and the consumption of new content, I’m time-shifting the curation process to a time better suited for reading, and (most critically) removed from the temptations, stresses, and biopsychosocial hooks that first lured me in. +

        +

        + I am always amazed by what happens: no matter how stringent I was in the original collecting, no matter how certain I was that this thing was worthwhile, I regularly eliminate 1/3 of my list before reading. The post that looked SO INTERESTING when compared to that one task I’d been procrastinating on, in retrospect isn’t even something I care about. +

        +

        + What I’m essentially doing is creating a buffer. Instead of pushing a new piece of info through from intake to processing to consumption without any scrutiny, I’m creating a pool of options drawn from a longer time period, which allows me to make decisions from a higher perspective, where those decisions are much better aligned with what truly matters to me. +

        +
        +

        + Remove any feature, process, or effort that does not directly contribute to the learning you seek. — Eric Ries, The Leader’s Guide +

        +
        +

        + Here’s a visual of how this works, from my Pocket analytics: +

        +

        + +

        +

        + You can see that I save more things toward the beginning of the week and the weekend, and then draw down the buffer more towards the end of the week. +

        +

        + /sidebar +

        +

        + Imagine for a second if we could do this with everything. On Saturday morning, well-rested and wise, you retroactively decide everything you want to have done during the previous week. Anything you decide was not worthwhile, you get that time back. +

        +

        + I experienced this recently with email — after returning from a 10-day meditation course during which I was completely off the grid, I was surprised to notice it took only 1.9 hours to process almost 2 weeks’ worth of email (I track these things). I normally spend on average 2.19 hours on email per week — what happened to those extra 2.48 hours?! Besides the gains from batch processing such a large quantity of emails at once, I believe the main factor was that I evaluated my emails from a longer time horizon and higher perspective, more correctly judging whether something was worth responding to or acting on. +

        +

        + If only this method would scale. +

        +

        + /end_sidebar +

        +

        + Mo’ apps, mo’ problems +

        +

        + There are drawbacks, which I’ve glossed over until now. The two main ones: +

        +

        + 1. Formatting issues +

        +

        + Many sites, including popular ones, aren’t presented correctly within the Pocket app (and I imagine others). There’s always the option of opening the link in a web browser, but this eliminates all the positive affordances and then some. If there wasn’t so much value provided otherwise, this would be a deal breaker. +

        +

        + The worst part is that, sometimes, the article is cut off or links don’t appear without any indication that something is amiss. On Tim Ferriss’ blog, for example, links (of which there are many) are simply removed. +

        +

        + One solution is to tag problematic items with “desktop” so you know that these need to be read/viewed on your computer. +

        +

        + 2. Dependence +

        +

        + Every productivity tool eventually becomes a victim of its own success. In this case, I’ve become so dependent on Pocket that bugs really affect me. +

        +

        + For example, the Share to Evernote feature, which I use to highlight and save key passages, has been broken for at least a month. My hysterical tweets to Pocket Support have been answered but not resolved. +

        +

        + You wouldn’t think such a minor feature within one app could be so disruptive, but it has been massively so. This simple workflow: +

        +

        + Highlight > Share > Share to Evernote > Save +

        +

        + …has been replaced with this: +

        +

        + Highlight > Copy > Switch to Evernote > New note > Paste > Switch back to Pocket > Share > More > Copy URL > Switch back to Evernote > Paste URL > Switch back to Pocket +

        +

        + Worse, I often forget to go back and grab the URL, so I have to hunt it down at some later date. +

        +

        + /rant_over +

        +

        + Progress Traps and Paradigms +

        +

        + The amount of information in the world is a progress trap. Too much stuff to read is just as limiting as too little. +

        +

        + As the inimitable Venkatesh Rao has written, we’re moving from a world of containers (companies, departments, semesters, packages, silos) to a world of streams (social networks, info feeds, main streets of thriving cities, Twitter). Problems and opportunities alike resist having neat little boxes drawn around them. There’s way too much to absorb. Way too much to even guess what you don’t know. +

        +

        + As the pace of change in the world accelerates, we double down on all the methods that created the problems in the first place — more planning, more forecasting, more control and risk management. We’re left with massive institutions that nobody trusts, that are simultaneously brittle and too-big-to-fail, creating precarity at every level of the socioeconomic pyramid. +

        +

        + What would it look like instead to solve problems (and explore opportunities) in a way that gets better the faster we go? +

        +

        + I can’t do justice to Rao’s blog series linked above (it’s in 20 parts — may want to save it for later ;), but the first step he proposes is “exposing yourself to as many different diverse streams as possible.” +

        +

        + When you’re immersed in a stream, the faster it goes, the more novel perspectives and ideas you’re exposed to. You develop an opposable mind — the ability to juggle and play around with different perspectives on any issue, instead of seeing it through one lens. +

        +

        + Increasingly, the only metric that will matter in your journey of personal growth will be ROL: Rate-of-Learning. We’ve heard a lot in recent years about the importance of hands-on learning and practical experimentation. We get it. Burying your head in a book by itself gets you nowhere. +

        +

        + But the pendulum is swinging too far in that direction. Yes, you can be too action-oriented. Ideas, while cheap when compared to effective execution, are still more valuable than many of the other things we spend time on. +

        +

        + There’s another way to learn faster: assimilate and build on the ideas of others. Sure, you won’t understand every tacit lesson their experience gave them, but you can incorporate many of them, and in a fraction of the time it would take you to make every mistake yourself. +

        +

        + Ideas are high leverage agents. They become more so when arranged in highly cross-referenced networks. The only tool we have available that is capable of both creating and accessing these networks on demand is the human brain. +

        +

        + I lied before. There is one form of leverage even more powerful than the initial assumptions and paradigms that inform a system’s development: the ability to transcend paradigms. +

        +

        + I can’t put it any better than Donella Meadows, in her seminal piece on complex systems: +

        +
        +

        + People who cling to paradigms (which means just about all of us) take one look at the spacious possibility that everything they think is guaranteed to be nonsense and pedal rapidly in the opposite direction. Surely there is no power, no control, no understanding, not even a reason for being, much less acting, in the notion or experience that there is no certainty in any worldview. But, in fact, everyone who has managed to entertain that idea, for a moment or for a lifetime, has found it to be the basis for radical empowerment. If no paradigm is right, you can choose whatever one will help to achieve your purpose. +

        +
        +
        +

        + It is in this space of mastery over paradigms that people throw off addictions, live in constant joy, bring down empires, get locked up or burned at the stake or crucified or shot, and have impacts that last for millennia. +

        +
        +
        +

        + In the end, it seems that mastery has less to do with pushing leverage points than it does with strategically, profoundly, madly letting go. +

        +
        +

        + Reading is the closest thing we have to thinking another’s thoughts. It’s long and sometimes ponderous, but that work is required to wrap yourself in another person’s paradigm. Which is the first step in madly letting go of your own. +

        +

        + The amazing thing about ideas is that it takes zero time for one to change your paradigm. It happens in time, but takes no time, like an inter-dimensional wormhole, one entangled particle in your brain mirroring its twin across a chasm even more vast than the universe — the chasm between two minds. +

        +

        + And that is the secret power of Read It Later apps. +

        +

        + P.S. My latest setup has 2 parts: 1) using this IFTTT recipe to automatically send “liked” articles in Instapaper to an Evernotebook called “Instapaper favorites” (for things I want to save in general but don’t have any particular notes on), and 2) this recipe that saves anything I highlight in Instapaper to a new note, and sends it to the Evernote default notebook where I can decide where it belongs later (for when I have specific passages I want to extract) +

        +
        + Subscribe below to receive free weekly emails with our best new content, or follow us on Twitter, Facebook, Instagram, LinkedIn, or YouTube. Or become a Praxis member to receive instant access to our full collection of members-only posts.
        +
        +
        +
        +
        +
        +
        +
        +

        + Join the Forte Labs Newsletter +

        +
        +
        +

        + Join 50,000+ people receiving my best ideas on learning, productivity & knowledge management every Tuesday. I''ll send you my Top 10 All-Time Articles right away as a thank you. +

        +
        +
        +
        +
          +
          +
          + +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +

          + Get my top-10 productivity tips +

          +
          +
          +
          +
          +
          + Subscribe to my free weekly newsletter and get my best tips on how to use technology to be more productive. +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + RESOURCES +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + ABOUT +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + CONNECT +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          + Twitter Facebook Youtube Linkedin Instagram +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + +
          +
            +
          • + +
            • +
          • + +
            • +
          • + +
            • +
          • + +
            • +
          • + +
            • +
          • + +
            • +
          +
          +
          +
          +
          + Share This +
          + +
          +
          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +', 3726), + ('elad_meetings', 'http://blog.eladgil.com/2018/07/meeting-etiquette.html', 'Better Meetings', 'Elad Gil', 'How to make meetings more productive.', null, '2018-07-02', 'Elad Blog', 'better-meetings', '
          +
          +

          As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
          +
          1. Determine who is necessary in the meeting.
          + Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
          +
          2. Send out an agenda in advance.
          + What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
          +
          Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
          +
          3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
          + If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
          +
          If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
          +
          4. Kick off the meeting with objectives.
          + Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
          +
          5. Assign a note taker.
          + Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
          +
          6. Send out meeting notes.
          + Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
          +
          Meeting notes would optimally include:
          + a. Subject/topic of meeting
          + b. Date
          + c. Attendees
          + d. Actions/decisions
          + e. Agenda
          + f. Detailed notes
          +
          7. Clean up the meeting calendar ongoing.
          + Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
          +
          NOTES
          + [1] There may be legal reasons to not take notes in specific meetings. Consult with the company''s GC or external counsel on training for meeting note etiquette. +

          +

          MY BOOK +

          +

          You can pre-order the High Growth Handbook here.
          +
          +

          +
          +
          +', ' + + + + + + + + + + + + + + + + + Elad Blog: Better Meetings + + + + + + + + + + + + + + + + + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +

          + Monday, July 2, 2018 +

          +
          +
          +
          + + +

          + Better Meetings +

          +
          +
          +
          +
          +
          + As a company scales the number of meetings initially grows faster then headcount. With more people comes more coordination. Most companies have bad meeting etiquette, which means an enormous amount of time is wasted. The following steps help increase meeting efficiency:
          +
          1. Determine who is necessary in the meeting.
          + Are there really 20 people needed in the room? Separate the must haves, from the nice to haves, from the politically expedient to be there.
          +
          2. Send out an agenda in advance.
          + What will be discussed? How much time do you really need to spend per topic? Maybe the 60 minute meeting should really be 30 minutes? Like wedding planning, meetings fill the time available to them.
          +
          Similarly, what preparation should people do in advance? Is there a document to pre-read or data to look at in advance so people come prepared?
          +
          3. Set up (projecting, hangout or conference line, etc.) in advance if you can.
          + If you are running the meeting and are able to do so, arrive in advance. Dial into the conference line or start the Hangout and start projecting. Instead of wasting 5 minutes in setup with 10 people there, do it early.
          +
          If your conference rooms are always fully booked back to back (often the case as the fast growing companies are always running out of space), it is hard to get into the room 5 minutes early. Companies can facilitate this by having an :05 policy. I.e. meetings start at 2:05 but the meeting owner books the room starting at 2:00 so has 5 minutes to set up.
          +
          4. Kick off the meeting with objectives.
          + Review the agenda and purpose of the meeting. Is it to make a decision on a product? To brainstorm a new feature? To review a sales pipeline and prioritize leads? If the meeting does not have a clear objective, you should cancel it. If the same topic is repeated over and over in a weekly meeting without progress, decide if escalation or another method of breaking a bottleneck is needed. The same meeting should not take place 5 times.
          +
          5. Assign a note taker.
          + Who is responsible for taking notes in the meeting? Any meeting with more then 3-4 people should send out notes. The meeting owner can assign/delegate note taker up front.
          +
          6. Send out meeting notes.
          + Meeting notes help the rest of the company know what was discussed or decided. It allows different stakeholders to follow up if they were unable to attend. Notes increase cross-company transparency dramatically [1].
          +
          Meeting notes would optimally include:
          + a. Subject/topic of meeting
          + b. Date
          + c. Attendees
          + d. Actions/decisions
          + e. Agenda
          + f. Detailed notes
          +
          7. Clean up the meeting calendar ongoing.
          + Unnecessary regular meetings accumulate like rust on a company. Ask all meeting owners to go through and kill meetings once a quarter. (a) What meetings are still necessary or useful? (b) Who should still attend? People can be dropped and added as well to rebalance who should attend.
          +
          NOTES
          + [1] There may be legal reasons to not take notes in specific meetings. Consult with the company''s GC or external counsel on training for meeting note etiquette.
          +
          +
          +
          +
          + MY BOOK +
          You can pre-order the High Growth Handbook here.
          +

          +
          +
          +
          +
          + RELATED POSTS +
          + + + + + + +
          +
          +
          +
          +
          + Posted by at + +
          + +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + + + + + + +
          + + + +
          + +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          +
          + + + + + + +', 545), + ('jonbo_digital_tools', 'https://jon.bo/posts/digital-tools/', 'Digital Tools I Wish Existed', 'Jonathan Borichevskiy', 'My digital life in a nutshell: I discover relevant content I don’t have time to consume...', null, '2019-11-28', 'JON.BO', 'digital-tools-i-wish-existed', '
          +
          +

          My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later.

          +

          There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed:

          +
            +
          • +

            The more seamless the acquisition & ingestion, the more engaged I am with the content

            +
            • +
          • +

            Insights are just as likely to be found in a 400-page book as in a 40-minute podcast

            +
            • +
          • +

            Notes and their subsequent review are essential for long-term retention

            +
            • +
          • +

            Recommendations from other humans are as good, if not better, than algorithmic suggestions

            +
            • +
          +

          In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here.

          +

          Queue management for inbound digital content # +

          +

          Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance:

          +
            +
          • +

            Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more.

            +
            • +
          • +

            Every book, article, post, or tweet has the potential to lead to more content.

            +
            • +
          • +

            Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs.

            +
            • +
          • +

            I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee?

            +
            • +
          • +

            Learning, work, news, and entertainment all have different priorities in my life (roughly in that order).

            +
            • +
          • +

            I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month.

            +
            • +
          • +

            I’m not always connected to a stable internet connection.

            +
            • +
          • +

            If it’s a long piece of content I want my position saved reliably so I can resume at a later point.

            +
            • +
          • +

            I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation.

            +
            • +
          • +

            I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place.

            +
            • +
          • +

            I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat.

            +
            • +
          +
          + +
          Relevant XKCD, as is tradition
          +
          +

          Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one.

          +

          Honorable Mentions: Pocket, Instapaper +

          +

          A universal book log, recommendation & sharing system # +

          +

          I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof.

          +

          Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry.

          +

          Whatever this tool for managing my ever-growing reading list will be, it should:

          +
            +
          • +

            Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests.

            +
            • +
          • +

            Allow me to tag books instead of placing them into static lists (think clusters or tag clouds).

            +
            • +
          • +

            Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to.

            +
            • +
          • +

            Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it.

            +
            • +
          • +

            Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was.

            +
            • +
          • +

            Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on.

            +
            • +
          • +

            Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from.

            +
            • +
          +

          Honorable Mentions: None :(

          +

          Intelligent PDF viewers, eBook readers, audiobook & podcast players # +

          +
          + +
          Functionality I want in my document reader
          +
          +

          Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to:

          +
            +
          • +

            Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them.

            +
            • +
          • +

            See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition.

            +
            • +
          • +

            View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know!

            +
            • +
          • +

            Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019.

            +
            • +
          • +

            Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues.

            +
            • +
          • +

            Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points).

            +
            • +
          • +

            Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading.

            +
            • +
          +
          + +
          What I want my audiobook player to look like
          +
          +

          Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could:

          +
            +
          • +

            Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street.

            +
            • +
          • +

            View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps?

            +
            • +
          +

          Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar +

          +

          A centralized search interface for my digital brain (memex) # +

          +

          I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything.

          +

          I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should:

          +
            +
          • +

            Accept and parse the following queries:

            +
              +
            • +

              spacex announcement type:video 2016

              +
              • +
            • +

              links from:jon@test.org topic:python

              +
              • +
            • +

              paper on temperature, productivity referenced in book:Uninhabitable Earth

              +
              • +
            • +

              type:pdf habits digital interfaces

              +
              • +
            • +

              reading comprehension type:blog post

              +
              • +
            • +

              printer ink receipt

              +
              • +
            • +

              type:book read:2017 finance

              +
              • +
            • +

              file:py datetime parse

              +
              • +
            +
            • +
          • +

            Respect my privacy: hosted on something I control and never mined for ads.

            +
            • +
          • +

            Support all my devices with two-way sync so I can search and add to it wherever I am.

            +
            • +
          • +

            Be extensible: allow me to easily ingest my own information and extend with desired functionality.

            +
            • +
          • +

            Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of.

            +
            • +
          • +

            Notify me about changes to documents and webpages I’ve visited.

            +
            • +
          • +

            Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it.

            +
            • +
          +

          Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop +

          +

          Parting Thoughts # +

          +

          I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools.

          +

          It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share.

          +

          I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback.

          +

          + HN Discussion +

          +

          Appendix # +

          +

          + The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS +

          +
          +
          ', ' + + + + Digital Tools I Wish Existed :: up & to the right — Jonathan Borichevskiy + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
          +
          + + + +
          +
          +
          +

          + Digital Tools I Wish Existed +

          +

          +
          + Published: 2019-11-28 +
          #memex  #information  #learning  #web  #internet  +
          +

          + My digital life in a nutshell: I discover relevant content I don’t have time to consume, I find time and become overwhelmed with my scattered backlog, I wish the content were in a different format, and then I’m unable to find something again once I’ve consumed it. Not retaining enough is a valid problem but we’ll tackle that one later. +

          +

          + There’s a lot of generalization in my summary but the core issue is an extraordinarily high level of friction in the process of finding, organizing, and sharing digital content. During the past few years I’ve noticed: +

          +
            +
          • +

            + The more seamless the acquisition & ingestion, the more engaged I am with the content +

            +
            • +
          • +

            + Insights are just as likely to be found in a 400-page book as in a 40-minute podcast +

            +
            • +
          • +

            + Notes and their subsequent review are essential for long-term retention +

            +
            • +
          • +

            + Recommendations from other humans are as good, if not better, than algorithmic suggestions +

            +
            • +
          +

          + In the rest of this post I attempt to explain the digital tools I wish existed, and how the the currently available tools do not suffice. What are also probably lacking are my habits and workflows around this - but I’m looking at tools specifically here. +

          +

          + Queue management for inbound digital content # +

          +

          + Where to begin? Probably the most common problem I see myself and other people dealing with is processing the incoming deluge of articles to read and videos to watch. This isn’t all personal recommendations - it encompasses any and all content I think my future self would appreciate me consuming. A list of issues, roughly by order of appearance: +

          +
            +
          • +

            + Content (or links to it) arrive from a variety of sources including text messages from friends, email conversations, tweetstorms and replies, references in books, suggestions in real-world conversations, and more. +

            +
            • +
          • +

            + Every book, article, post, or tweet has the potential to lead to more content. +

            +
            • +
          • +

            + Content is published in a variety of formats including but not limited to images, sound files, videos, Google Drive docs, diagrams, long-form paywalled articles, PDFs, powerpoint presentations, and base 64 encoded blobs. +

            +
            • +
          • +

            + I have little visibility into required time investment and foundational context until I’ve opened it and started thinking about it. Should I sit down with a pen and paper to read this or can I skim it while waiting for my coffee? +

            +
            • +
          • +

            + Learning, work, news, and entertainment all have different priorities in my life (roughly in that order). +

            +
            • +
          • +

            + I would like to batch process content in different “streams” regardless of where they are stored. For example: I have two hours, let me work through interesting text content my friends sent me last week. Or: show me all the interesting/relevant videos I’ve queued over the past month. +

            +
            • +
          • +

            + I’m not always connected to a stable internet connection. +

            +
            • +
          • +

            + If it’s a long piece of content I want my position saved reliably so I can resume at a later point. +

            +
            • +
          • +

            + I often want it in a different format than the one it was originally published in (audio → text, text → audio, pdf → ebook). Automated conversion works but is cumbersome. Listening to text articles requires sending them to a special app and converting articles to ebooks is annoying and loses a lot of formatting and navigation. +

            +
            • +
          • +

            + I love to respond to a person’s recommendation - preferably before they’ve forgotten why they sent me it in the first place. +

            +
            • +
          • +

            + I’d like a centralized history of content tied to my notes and annotations in case I want to find it again later. It feels like every week I’m speaking with someone and I remember a blog post I read a few months ago they might find relevant … or was it a Reddit post? Can I find it my history? Oh no, it’s been replaced with [deleted] … find an archived copy… rinse and repeat. +

            +
            • +
          +
          + +
          + Relevant XKCD, as is tradition +
          +
          +

          + Following my curiosity feels like chasing a caffeinated bunny around while real understanding requires time, perspective, and reflection. The internet makes the former much easier - so I find myself constantly balancing the two. Additionally, my energy and attention levels vary throughout the day and it’s far easier to just open Twitter rather than continue reading a long-form article I started on my laptop two days ago. Too often I default to the lower-friction one. +

          +

          + Honorable Mentions: Pocket, Instapaper +

          +

          + A universal book log, recommendation & sharing system # +

          +

          + I love exploring other peoples’ reading lists. Here’s my own. I find everyone keeps their reading lists in different formats on different platforms. Plaintext lists are nice but hard to parse. Spreadsheets are easy to parse but a pain to manage. Third-party services aren’t interoperable, require logins, and are not future-proof. +

          +

          + Part of the problem here is metadata is hard. Someone has to sit there and fill out the author, title, subtitle, summary, page count - and they’re probably not going to do it for free. Amazon is a good at it but is hostile to publishers. Goodreads has much potential but seems to have stagnated. Linking to the book’s Wikipedia entry would be my preference but very few books have an entry. +

          +

          + Whatever this tool for managing my ever-growing reading list will be, it should: +

          +
            +
          • +

            + Let me compare my reading list with another to see overlap. I find this a wonderful way to spark conversation and find common interests. +

            +
            • +
          • +

            + Allow me to tag books instead of placing them into static lists (think clusters or tag clouds). +

            +
            • +
          • +

            + Be tied to my highlights, annotations, and bookmarks in a non-proprietary, searchable, and shareable format. Make them public if I want to. +

            +
            • +
          • +

            + Save context on where and when I found this book: why I thought it was important to read, when I read it, what I wrote down while reading it, and what other content I discovered through it. +

            +
            • +
          • +

            + Let me query this tool like a relational database. For example: show me all books about scaling startups recommended by people I follow on Twitter or by people they follow. The current Twitter search makes me feel like I’m using a government site created before I myself even knew what a computer was. +

            +
            • +
          • +

            + Help me deal with prioritization. My reading list is a mess and I can’t be alone. Are certain books better read before others? Prerequisites? Could three of them be replaced with one? What are the other books by the this author? Are they worth reading too? Why exactly did I think reading this 800 page book was relevant when I added it? Is 80% of the content attainable from a blog post? Where is that post? Has someone in my network written a rebuttal to the ideas in this book? The list goes on and on. +

            +
            • +
          • +

            + Provide relevant suggestions with the typical recommender approach based on what people interested in the same topics also enjoyed reading and learning from. +

            +
            • +
          +

          + Honorable Mentions: None :( +

          +

          + Intelligent PDF viewers, eBook readers, audiobook & podcast players # +

          +
          + +
          + Functionality I want in my document reader +
          +
          +

          + Reading is incredible and I love my Kindle. But eBooks today are just a step above OCR’ing a book and slapping on a few basic features which have existed for 30+ years. While I’m reading an eBook I want to: +

          +
            +
          • +

            + Have relevant illustrations, graphs, and tables appear for duration of their mentions so I don’t have to flip back and forth between them. +

            +
            • +
          • +

            + See glossary terms and their definitions which appear on this page. Highlighting and searching a term is great but the author may have added important context to the glossary definition. +

            +
            • +
          • +

            + View popular annotations and highlights across all mediums - not just by other readers who own an Amazon Kindle readers and purchased this book version and also happened to highlight it enough times. A quote was referenced in 300 blog articles? A two sentence excerpt retweeted 50,000 times? You bet I want to know! +

            +
            • +
          • +

            + Follow referenced information easily. You cited a paper - great, let’s look at the footnotes. Oh, the full reference is in the back of the book. Online list of citations? Of course not! Drop a bookmark, navigate to the back of the book, pull out my laptop, find the paper. Of course, a paywall. Grab a snack. Acquire the PDF. Search for keywords to try to find the referenced information. Sigh, 2019. +

            +
            • +
          • +

            + Not be hindered by the DRM system. Copyright is important and I want to support authors but it’s insane to me all these content licenses I’m acquiring can’t be donated to a library upon account closure. Yes, legal DRM-free eBooks exist but they aren’t without their own issues. +

            +
            • +
          • +

            + Seamlessly switch between devices and formats while retaining my position. Something like Whispersync (a neat idea but come on, I’m not made of money. Also, see above points). +

            +
            • +
          • +

            + Let me use a digital or physical keyboard instead of an e-ink keyboard to type my annotations. A possibility here is a companion app, which feels like a notes app but ties my notes to their location/text in the book I’m reading. +

            +
            • +
          +
          + +
          + What I want my audiobook player to look like +
          +
          +

          + Most of these points above also apply to my experience listening to podcasts, audiobooks, and watching Youtube videos and interviews. I find myself wishing I could: +

          +
            +
          • +

            + Navigate them more comfortably. Both Libby and Audible leave much to be desired in terms of navigation. Finding a quote I remember hearing to three days ago is basically blindly stumbling around - and I lose my current spot too. Seeing a list of chapter numbers for the book I’m listening to has been helpful a grand total of 0 times. And how cool would it be to drop a bookmark from my bluetooth-connected headphones as I’m biking down a street. +

            +
            • +
          • +

            + View an auto-generated transcript of a podcast as I’m listening to it. It should have easy-to-follow links to references to other podcasts, media, books and support searching for key terms. YouTube already transcribes all of their videos and Google Meet now generates live captions as we’re talking - why can’t we do something similar with podcast apps? +

            +
            • +
          +

          + Honorable Mentions: Readwise, Weava, Descript, Otter.ai, Polar +

          +

          + A centralized search interface for my digital brain (memex) # +

          +

          + I want to be able to open an interface, type three words, and instantly see results from everything my digital self has interacted with. Emails, years of full-text browsing history, text messages, Slack messages across all my organizations, calendar invites and events, books, podcast transcripts I’ve consumed, Twitter and Instagram DMs, PDFs I’ve downloaded, bash commands, videos I’ve seen, my online and offline files, notes, blog post drafts - I really do mean everything. +

          +

          + I acutely feel the need for this when I’m trying to find something I know I’ve seen online but can’t remember where I saw it. Google is wonderful for finding new information, but absolutely poor for re-finding things. Chrome’s history has so much potential - but I suspect Google would much rather have us look at their ads a few additional times rather than go direct to the source. I accept I might be in the minority on this one. Regardless, this tool should: +

          +
            +
          • +

            + Accept and parse the following queries: +

            +
              +
            • +

              + spacex announcement type:video 2016 +

              +
              • +
            • +

              + links from:jon@test.org topic:python +

              +
              • +
            • +

              + paper on temperature, productivity referenced in book:Uninhabitable Earth +

              +
              • +
            • +

              + type:pdf habits digital interfaces +

              +
              • +
            • +

              + reading comprehension type:blog post +

              +
              • +
            • +

              + printer ink receipt +

              +
              • +
            • +

              + type:book read:2017 finance +

              +
              • +
            • +

              + file:py datetime parse +

              +
              • +
            +
            • +
          • +

            + Respect my privacy: hosted on something I control and never mined for ads. +

            +
            • +
          • +

            + Support all my devices with two-way sync so I can search and add to it wherever I am. +

            +
            • +
          • +

            + Be extensible: allow me to easily ingest my own information and extend with desired functionality. +

            +
            • +
          • +

            + Cluster information based on content, tags, geo-location, connected people, conversations, source, and other factors I’m not even aware of. +

            +
            • +
          • +

            + Notify me about changes to documents and webpages I’ve visited. +

            +
            • +
          • +

            + Allow a rough export of my research on a topic (like, a knowledge dump off everything I’ve consumed on pandas) with the ability to easily share it. +

            +
            • +
          +

          + Honorable Mentions: Memex by Worldbrain.io, Roam Research, Notion, Coda.io, Alfred, Trove, Local Native, ArchiveBox, Raindrop +

          +

          + Parting Thoughts # +

          +

          + I’m fascinated with a better bridge between our minds and our digital devices. A well-designed tool should disappear and allow complete attention to the task at hand, but digital devices today are far from this ideal - often due to arcane copyright laws or profit-seeking. These aren’t new ideas by any means. See Vannevar Bush’s original conception of a memex over 70 years ago. We are way overdue for this. I see enormous potential at combining a true memex with all of our personal data (health, fitness, biometrics) along with our habits, goals, tasks, reflections, and communication tools. +

          +

          + It seems to me that as information becomes more abundant, the connections drawn between disparate pieces are becoming increasingly important. The easier it is to share that graph with other people, the faster we can learn from each other and understand complex relationships. I’m excited for a world where knowledge is easier to discover, validate, dispute, understand, retain, and share. +

          +

          + I hope to cover my thoughts on processes, note-taking apps, and knowledge graphs next. Stay tuned here. My thanks to Arthur Tyukayev, Alex Ly, David Heimann, Em deGrandpré, Alexey Guzey, Sam Tkachuk, and Brian Timar for reading drafts of this and providing wonderful feedback. +

          +

          + HN Discussion +

          +

          + 2019-12-09: fixes grammar +

          +

          + Appendix # +

          +

          + The sad state of personal data and infrastructure (beepb00p.xyz) Note-Taking when Reading the Web and RSS +

          +
          + +
          +
          + + + +
          + + +', 2285); + +COMMIT; diff --git a/packages/db/migrations/0140.undo.popular_read.sql b/packages/db/migrations/0140.undo.popular_read.sql new file mode 100755 index 000000000..c45addf8a --- /dev/null +++ b/packages/db/migrations/0140.undo.popular_read.sql @@ -0,0 +1,9 @@ +-- Type: UNDO +-- Name: popular_read +-- Description: Create omnivore.popular_read table + +BEGIN; + +DROP TABLE IF EXISTS omnivore.popular_read; + +COMMIT; diff --git a/packages/db/package.json b/packages/db/package.json index bf7cfd766..fce36c4af 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -9,7 +9,6 @@ "author": "", "license": "ISC", "dependencies": { - "@elastic/elasticsearch": "~7.12.0", "dotenv": "^8.2.0", "pg": "^8.3.0", "postgrator": "^4.1.1", diff --git a/packages/pdf-handler/src/pdf.ts b/packages/pdf-handler/src/pdf.ts index 48d85b637..0585b742b 100644 --- a/packages/pdf-handler/src/pdf.ts +++ b/packages/pdf-handler/src/pdf.ts @@ -14,7 +14,7 @@ interface Page { lines: string[] } -// Unused at the moment -- comented out for now to satisfy linter +// Unused at the moment -- commented out for now to satisfy linter const MAX_TITLE_LENGTH = 95 type MetadataInfoKey = diff --git a/packages/puppeteer-parse/README.md b/packages/puppeteer-parse/README.md index 6155c45b0..373c57353 100644 --- a/packages/puppeteer-parse/README.md +++ b/packages/puppeteer-parse/README.md @@ -1,3 +1,3 @@ # Puppeteer parsing function handler -This workspace is used to provide the module for the app to hande requests for the article parsing via Puppeteer. +This workspace is used to provide the module for the app to handle requests for the article parsing via Puppeteer. diff --git a/packages/puppeteer-parse/index.js b/packages/puppeteer-parse/index.js index b3413a244..64f8b91a7 100644 --- a/packages/puppeteer-parse/index.js +++ b/packages/puppeteer-parse/index.js @@ -40,7 +40,7 @@ const DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) Apple const BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_DESKTOP_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_6_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4372.0 Safari/537.36' const NON_BOT_HOSTS = ['bloomberg.com', 'forbes.com'] -const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com']; +const NON_SCRIPT_HOSTS= ['medium.com', 'fastcompany.com', 'fortelabs.com']; const ALLOWED_CONTENT_TYPES = ['text/html', 'application/octet-stream', 'text/plain', 'application/pdf']; @@ -272,7 +272,7 @@ const sendSavePageMutation = async (userId, input) => { } }`, variables: { - input: Object.assign({}, input , { source: 'puppeteer-parse' }), + input, }, }); @@ -341,7 +341,7 @@ async function fetchContent(req, res) { const articleSavingRequestId = (req.query ? req.query.saveRequestId : undefined) || (req.body ? req.body.saveRequestId : undefined); const state = req.body.state const labels = req.body.labels - const source = req.body.source || 'parseContent'; + const source = req.body.source || 'puppeteer-parse'; const taskId = req.body.taskId; // taskId is used to update import status const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined); const locale = (req.query ? req.query.locale : undefined) || (req.body ? req.body.locale : undefined); @@ -473,6 +473,7 @@ async function fetchContent(req, res) { rssFeedUrl, savedAt, publishedAt, + source, }); if (!apiResponse) { logRecord.error = 'error while saving page'; diff --git a/packages/queue-manager/.eslintignore b/packages/queue-manager/.eslintignore new file mode 100644 index 000000000..b38db2f29 --- /dev/null +++ b/packages/queue-manager/.eslintignore @@ -0,0 +1,2 @@ +node_modules/ +build/ diff --git a/packages/queue-manager/.eslintrc b/packages/queue-manager/.eslintrc new file mode 100644 index 000000000..e006282a6 --- /dev/null +++ b/packages/queue-manager/.eslintrc @@ -0,0 +1,6 @@ +{ + "extends": "../../.eslintrc", + "parserOptions": { + "project": "tsconfig.json" + } +} \ No newline at end of file diff --git a/packages/queue-manager/Dockerfile b/packages/queue-manager/Dockerfile new file mode 100644 index 000000000..fe547f483 --- /dev/null +++ b/packages/queue-manager/Dockerfile @@ -0,0 +1,26 @@ +FROM node:14.18-alpine + +# Run everything after as non-privileged user. +WORKDIR /app + +COPY package.json . +COPY yarn.lock . +COPY tsconfig.json . +COPY .eslintrc . + +COPY /packages/queue-manager/package.json ./packages/queue-manager/package.json + +RUN yarn install --pure-lockfile + +ADD /packages/queue-manager ./packages/queue-manager +RUN yarn workspace @omnivore/queue-manager build + +# After building, fetch the production dependencies +RUN rm -rf /app/packages/queue-manager/node_modules +RUN rm -rf /app/node_modules +RUN yarn install --pure-lockfile --production + +EXPOSE 8080 + +CMD ["yarn", "workspace", "@omnivore/queue-manager", "start"] + diff --git a/packages/queue-manager/mocha-config.json b/packages/queue-manager/mocha-config.json new file mode 100644 index 000000000..44d1d24c1 --- /dev/null +++ b/packages/queue-manager/mocha-config.json @@ -0,0 +1,5 @@ +{ + "extension": ["ts"], + "spec": "test/**/*.test.ts", + "require": "test/babel-register.js" + } \ No newline at end of file diff --git a/packages/queue-manager/package.json b/packages/queue-manager/package.json new file mode 100644 index 000000000..7c1319a45 --- /dev/null +++ b/packages/queue-manager/package.json @@ -0,0 +1,32 @@ +{ + "name": "@omnivore/queue-manager", + "version": "1.0.0", + "main": "build/src/index.js", + "files": [ + "build/src" + ], + "license": "Apache-2.0", + "scripts": { + "test": "yarn mocha -r ts-node/register --config mocha-config.json", + "lint": "eslint src --ext ts,js,tsx,jsx", + "compile": "tsc", + "build": "tsc", + "start": "functions-framework --target=queueManager", + "dev": "concurrently \"tsc -w\" \"nodemon --watch ./build/ --exec npm run start\"" + }, + "devDependencies": { + "@types/node-fetch": "^2.6.6", + "chai": "^4.3.6", + "eslint-plugin-prettier": "^4.0.0", + "mocha": "^10.0.0" + }, + "dependencies": { + "@google-cloud/functions-framework": "3.1.2", + "@google-cloud/monitoring": "^4.0.0", + "@google-cloud/tasks": "^4.0.0", + "@sentry/serverless": "^6.16.1", + "axios": "^1.4.0", + "dotenv": "^16.0.1", + "jsonwebtoken": "^8.5.1" + } +} diff --git a/packages/queue-manager/src/index.ts b/packages/queue-manager/src/index.ts new file mode 100644 index 000000000..9574d0684 --- /dev/null +++ b/packages/queue-manager/src/index.ts @@ -0,0 +1,196 @@ +import { MetricServiceClient } from '@google-cloud/monitoring' +import { v2beta3 } from '@google-cloud/tasks' +import fetch from 'node-fetch' +import * as dotenv from 'dotenv' + +import * as Sentry from '@sentry/serverless' + +dotenv.config() +Sentry.GCPFunction.init({ + dsn: process.env.SENTRY_DSN, + tracesSampleRate: 0, +}) + +const PROJECT_ID = process.env.GCP_PROJECT_ID +const LOCATION = 'us-west2' +const IMPORT_QUEUE_NAME = process.env.IMPORT_QUEUE_NAME +const RSS_QUEUE_NAME = process.env.RSS_FEED_QUEUE_NAME +const QUEUE_NAMES = [IMPORT_QUEUE_NAME, RSS_QUEUE_NAME] +const DISCORD_WEBHOOK_URL = process.env.DISCORD_WEBHOOK_URL +const METRICS_FILTER = `metric.type="appengine.googleapis.com/http/server/response_latencies" metric.labels.response_code="200"` + +if ( + !PROJECT_ID || + !IMPORT_QUEUE_NAME || + !RSS_QUEUE_NAME || + !DISCORD_WEBHOOK_URL +) { + throw new Error('environment not supplied.') +} + +const LATENCY_THRESHOLD = 500 +const RSS_QUEUE_THRESHOLD = 20_000 +const IMPORT_QUEUE_THRESHOLD = 250_000 + +const postToDiscord = async (message: string) => { + console.log('notify message', { message }) + const payload = { + content: message, + } + + try { + const response = await fetch(DISCORD_WEBHOOK_URL, { + method: 'POST', + body: JSON.stringify(payload), + headers: { + 'Content-Type': 'application/json', + }, + }) + + if (!response.ok) { + throw new Error(`Discord response was not ok: ${response.statusText}`) + } + } catch (error) { + console.error('Failed to post message to Discord:', error) + } +} + +const checkShouldPauseQueues = async () => { + const now = Date.now() + const client = new MetricServiceClient() + + // Query for the metrics from the last 5 minutes + const [timeSeries] = await client.listTimeSeries({ + name: client.projectPath(PROJECT_ID), + filter: METRICS_FILTER, + interval: { + startTime: { + seconds: Math.floor(now / 1000 - 5 * 60), + }, + endTime: { + seconds: Math.floor(now / 1000), + }, + }, + aggregation: { + alignmentPeriod: { + seconds: 300, + }, + perSeriesAligner: 'ALIGN_PERCENTILE_95', + }, + }) + + for (const ts of timeSeries) { + // We only want to look at the backend service right now + if ( + !ts.resource || + !ts.resource.labels || + !ts.resource.labels['module_id'] || + !ts.resource.labels['module_id'].startsWith('backend') + ) { + continue + } + + if (ts.points && ts.points.length) { + const avgLatency = + ts.points.reduce( + (acc, point) => acc + (point.value?.doubleValue ?? 0), + 0 + ) / ts.points.length + if (avgLatency > LATENCY_THRESHOLD) { + return { shouldPauseQueues: true, avgLatency: avgLatency } + } + } + } + + return { shouldPauseQueues: false, avgLatency: 0 } +} + +const getQueueTaskCount = async (queueName: string) => { + const cloudTasksClient = new v2beta3.CloudTasksClient() + const queuePath = cloudTasksClient.queuePath(PROJECT_ID, LOCATION, queueName) + const [queue] = await cloudTasksClient.getQueue({ + name: queuePath, + readMask: { paths: ['name', 'stats'] }, + }) + + console.log(' queue.stats', { stats: queue.stats }) + if (Number.isNaN(queue.stats?.tasksCount)) { + return 0 + } + return Number(queue.stats?.tasksCount) +} + +const pauseQueues = async () => { + const cloudTasksClient = new v2beta3.CloudTasksClient() + + await Promise.all([ + cloudTasksClient.pauseQueue({ + name: cloudTasksClient.queuePath(PROJECT_ID, LOCATION, RSS_QUEUE_NAME), + }), + cloudTasksClient.pauseQueue({ + name: cloudTasksClient.queuePath(PROJECT_ID, LOCATION, IMPORT_QUEUE_NAME), + }), + ]) +} + +async function checkMetricsAndPauseQueues() { + if ( + !PROJECT_ID || + !IMPORT_QUEUE_NAME || + !RSS_QUEUE_NAME || + !DISCORD_WEBHOOK_URL + ) { + throw new Error('environment not supplied.') + } + + const { shouldPauseQueues, avgLatency } = await checkShouldPauseQueues() + + if (shouldPauseQueues) { + let rssQueueCount: number | string = 'unknown' + let importQueueCount: number | string = 'unknown' + try { + rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + } catch (err) { + console.log('error fetching queue counts', err) + } + + const message = `Both queues have been paused due to API latency threshold exceedance (${avgLatency}).\n\t-The RSS queue currently has ${rssQueueCount} tasks.\n\t-The import queue currently has ${importQueueCount} pending tasks.` + + await pauseQueues() + await postToDiscord(message) + } else { + try { + const rssQueueCount = await getQueueTaskCount(RSS_QUEUE_NAME) + const importQueueCount = await getQueueTaskCount(IMPORT_QUEUE_NAME) + + if (rssQueueCount > RSS_QUEUE_THRESHOLD) { + await postToDiscord( + `The RSS queue has exceeded it's threshold, it has ${rssQueueCount} items in it.` + ) + } + + if (importQueueCount > IMPORT_QUEUE_THRESHOLD) { + await postToDiscord( + `The import queue has exceeded it's threshold, it has ${importQueueCount} items in it.` + ) + } + } catch (err) { + console.log('error getting queue counts') + } + } +} + +export const queueManager = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + try { + if (req.query['check']) { + await checkMetricsAndPauseQueues() + } + res.send('ok') + } catch (e) { + console.error('Error while parsing RSS feed', e) + res.status(500).send('INTERNAL_SERVER_ERROR') + } + } +) diff --git a/packages/queue-manager/test/babel-register.js b/packages/queue-manager/test/babel-register.js new file mode 100644 index 000000000..a6f65f60a --- /dev/null +++ b/packages/queue-manager/test/babel-register.js @@ -0,0 +1,3 @@ +const register = require('@babel/register').default + +register({ extensions: ['.ts', '.tsx', '.js', '.jsx'] }) diff --git a/packages/rss-handler/test/stub.test.ts b/packages/queue-manager/test/stub.test.ts similarity index 100% rename from packages/rss-handler/test/stub.test.ts rename to packages/queue-manager/test/stub.test.ts diff --git a/packages/queue-manager/tsconfig.json b/packages/queue-manager/tsconfig.json new file mode 100644 index 000000000..7ebe093f6 --- /dev/null +++ b/packages/queue-manager/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "./../../tsconfig.json", + "compilerOptions": { + "outDir": "build", + "rootDir": "." + }, + "include": ["src"] +} diff --git a/packages/readabilityjs/test/index.html b/packages/readabilityjs/test/index.html index e997271df..a5c572757 100644 --- a/packages/readabilityjs/test/index.html +++ b/packages/readabilityjs/test/index.html @@ -14,22 +14,10 @@ diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected-metadata.json b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected-metadata.json new file mode 100644 index 000000000..f15f6cdab --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected-metadata.json @@ -0,0 +1,12 @@ +{ + "title": "Progressive Summarization: A Practical Technique for Designing Discoverable Notes", + "byline": "Tiago Forte", + "dir": null, + "excerpt": "Modern digital tools make it easy to “capture” information from a wide variety of sources. We know how to snap a picture, type out some notes, record a video, or scan a document. Getting this content from the outside world into the digital world is trivial. It’s even easier to get content that is already digital from one app to another. We know how to copy and paste text, save an image from a webpage, archive an email attachment, or import a video file.", + "siteName": "Forte Labs", + "siteIcon": "https://fortelabs.com/wp-content/uploads/2020/02/cropped-cropped-Icon_Red-1-32x32.png", + "previewImage": "https://fortelabs.com/wp-content/uploads/2017/12/1vTBT177SuhcgBEdPJwT9lA.jpeg", + "publishedDate": "2017-12-27T14:59:00.000Z", + "language": "English", + "readerable": true +} diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html new file mode 100644 index 000000000..99b0e8f9c --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/expected.html @@ -0,0 +1,245 @@ +
          +
          +
          +
          +
          +
          + Series Navigation: Progressive Summarization +
          + +
          +
          +

          + +

          +

          Modern digital tools make it easy to “capture” information from a wide variety of sources. We know how to snap a picture, type out some notes, record a video, or scan a document. Getting this content from the outside world into the digital world is trivial.

          +

          It’s even easier to get content that is already digital from one app to another. We know how to copy and paste text, save an image from a webpage, archive an email attachment, or import a video file.

          +

          What is difficult is not transferring content from place to place, but transferring it through time.

          +

          You know what I mean: you read a book, investing hours of mental labor in understanding the ideas it presents. You finish the book with a feeling of triumph that you’ve gained a valuable body of knowledge.

          +

          But then what?

          +

          You may try to apply the science-based methods the book recommends, only to realize it’s not quite as clear-cut as you thought. You may try to change the way you eat, exercise, communicate, or work, trusting in the power of habits. But then the everyday demands of life come rushing back, and you forget what motivated you in the first place.

          +

          At this point, people take different paths. Some give up, labeling all “self-help” books a waste of time. Others decide it’s just a problem of remembering everything they read, and invest in fancy memorization techniques. And many people become “infovores,” force-feeding themselves endless books, articles, and courses, in the hope that something will stick.

          +

          I want to suggest an alternative to all the approaches above: what you read is good and useful and very important, you’re just reading it at the wrong time.

          +

          You’re reading about time management techniques now, but they will only be useful two years from now, when you become a manager and have much greater demands on your time.

          +

          You’re watching YouTube videos on online marketing now, but that knowledge can only be put to use in 9 months, when your new online course gets off the ground.

          +

          You’re talking to a prospect about his goals and challenges now, but when you could really use that information is next year, when he is taking bids for a huge new contract.

          +

          The challenge of knowledge is not acquiring it. In our digital world, you can acquire almost any knowledge at almost any time.

          +

          The challenge is knowing which knowledge is worth acquiring. And then building a system to forward bits of it through time, to the future situation or problem or challenge where it is most applicable, and most needed.

          +

          At that future point, when you’re applying that knowledge directly to a real-world challenge, you won’t have to worry about memorizing it, integrating it, or even fully understanding it. You will only have to apply it, and any gaps in your understanding will very quickly reveal themselves. By the time you’re done solving a real problem with it, book knowledge has become experiential knowledge. And experiential knowledge is something you carry with you forever.

          +

          This is the job of a “second brain” — an external, integrated digital repository for the things you learn and the resources from which they come. It is a storage and retrieval system, packaging bits of knowledge into discrete packets that can be forwarded to various points in time to be reviewed, utilized, or deleted.

          +
          +
          +

          +

          Use Progressive Summarization
          to create easy-to-review notes

          +

          +

          I'll send you my Progressive Summarization Cheat Sheet as a thank you when you subscribe to my free weekly newsletter below.

          +
            +

            Look out for an email from hello@fortelabs.co

            +
            +
            +

            In The PARA Method, I described a universal system for organizing any kind of digital information from any source. It is a “good enough” system, maintaining notes according to their actionability (which takes just a moment to determine), instead of their meaning (which is ambiguous and depends on the context).

            +

            The four top-level categories of PARA — Projects, Areas, Resources, and Archives — are designed to facilitate this process of forwarding knowledge through time.

            +
              +
            • By placing a note in a project folder, you are essentially scheduling it for review on the short time horizon of an individual project
              • +
            • Notes in area folders are scheduled for less frequent review, whenever you evaluate that area of your work or life
              • +
            • Notes in resource folders stand ready for review if and when you decide to take action on that topic
              • +
            • And notes in archive folders are in “cold storage,” available if needed but not scheduled for review at any particular time
              • +
            +

            Note that we have re-created the tickler file, except instead of strict time-based horizons (daily, weekly, monthly, annually), they are scheduled contingently — if X happens, when Y arrives, if I want to do Z, etc.

            +

            + +

            +

            Planning in terms of contingencies gives us all the benefits of planning and researching, without locking us into rigid routines. We have the ability to massively accelerate, using our repository of accumulated notes as rocket fuel. But the actual decision of whether or not to accelerate, and critically, in which direction, we leave to our Future Self, who is older and wiser.

            +

            PARA answers how these “packets of knowledge” are organized: in discrete notes, sorted into 4 categories according to actionability, and resurfaced using RandomNote.

            +

            But now we turn to a more fundamental question: how are these packets made? Once we capture something, how do we structure the note so that it’s easily discoverable and usable in the future? How do we make sure what we’re saving today adds value to future projects, even when we can’t predict or even imagine what those projects might be?

            +

            That is the job of Progressive Summarization.

            +

            Note-first knowledge management

            +

            There are two primary schools of thought on how to organize a note-taking program (or really any body of information, but I’ll use terms specific to note-taking apps):

            +

            + +

            +

            + Tagging-first approaches argue that there should be no explicit hierarchy of notes, notebooks, and stacks. Notes are envisioned as an ever-changing, virtual matrix of interconnected, free-floating ideas. Because many tags can be applied to one note, there are multiple pathways to discover any given note. Locating notes in specific notebooks and folders is seen as limiting and static. +

            +

            + +

            +

            Although tags have their uses, I don’t believe they work as a primary organizational system. In my experience, relying on tagging is too fragile and requires too much maintenance, spreading attention too uniformly across all notes whether or not they are truly valuable. The virtual matrix sounds cool and futuristic, but our minds are not made to work well with such abstract concepts — we understand placing one thing in one place intuitively and automatically.

            +

            The second conventional approach to organizing notes is notebook-first. This basically translates how we organize things in the physical world — in a series of discrete containers — into the digital world.

            +

            + +

            +

            Notebook-first is better than tagging-first, in my opinion, mostly because it stays out of the way. It doesn’t try to automate and encroach upon the deeply intuitive act of making connections and seeing patterns. PARA on its own is a notebook-first system.

            +

            But if we stopped there, it would still be woefully inadequate for an economy based on creative output. As the tagging enthusiasts correctly point out, notebooks and folders actually suppress the serendipity and randomness that is at the heart of a creative lifestyle.

            +

            I propose a way to break the impasse: a note-first approach.

            +

            + +

            +

            I propose we make the design of individual notes the primary factor, instead of tags or notebooks. This has many advantages:

            +
              +
            • It works well with any other organizational system, without depending on them (including but not limited to tags and notebooks, if you want to use those)
              • +
            • It makes all work you do on your notes value-added, because you’re spending close to 100% of the time engaging directly with the content itself
              • +
            • It can more easily survive migrations to other devices, storage locations, and even programs, because note content is much more likely to be preserved than overarching structure
              • +
            • It cultivates skills (succinct communication, finding the core of an idea, visual thinking, etc.) that are inherently valuable and highly transferrable to other activities
              • +
            • It makes your notes more legible and useful to others (unlike your internal notebook structure, which is only for your use), promoting collaboration and sharing
              • +
            +

            With a note-first approach, your notes become like individual atoms — each with its own unique properties, but ready to be assembled into elements, molecules, and compounds that are far more powerful.

            +

            A note-first approach to knowledge management means we have to think about design. You are, in a very real sense, designing a product for a demanding customer — Future You.

            +

            Future You doesn’t necessarily trust that everything Past You put into your notes is valuable. Future You is impatient and skeptical, demanding proof upfront that the time they spend reviewing notes will be worthwhile. You’ve gotta “sell them” on the idea of reviewing a given note, including all the stages any salesperson has to master: gaining attention, inspiring interest, establishing credibility, stoking desire, and making a case for action NOW.

            +

            As if all that wasn’t intimidating enough, you have to do this for every single note without spending any extra time. You don’t have extra time, do you?

            +

            Let’s start at the beginning: at the heart of every design, we are trying to balance priorities. You want one thing, but it has to be balanced against something else that you also want.

            +

            You want a vehicle to protect its occupants, but you can’t just add layers and layers of titanium armor plating. You have to balance safety against weight and cost.

            +

            You want a phone to have the longest possible battery life, but you can’t just give it a 10-pound brick of a battery. You have to balance battery life against size and usability.

            +

            In the case of notes, I believe the two priorities we are trying to balance are discoverability and understanding.

            +

            + +

            +

            Making a note discoverable involves making it small, simple, and easy to digest. We accomplish this using compression: creating highly condensed summaries, without all the fluff.

            +

            But we also want to make our notes understandable. This involves including all the context: the details, the examples, and cited sources to be sure nothing falls through the cracks.

            +

            + +

            +

            This is a difficult tradeoff because you cannot compress something without losing some of its context.

            +

            You cannot summarize an article without discarding most of its points. You cannot make a highlight reel of a video without cutting out most of the footage. You cannot give an 18-minute TED talk without leaving out most of your ideas.

            +

            In making decisions about what to keep, you are inevitably making decisions about what to throw away.

            +

            Compression vs. context

            +

            There’s a natural tension between the two, compression and context.

            +

            + +

            +

            To communicate anything, you have to compress it, like communicating a huge amount of life experience in a wise saying. But in doing so, you lose a lot of the context that made that wisdom valuable in the first place.

            +

            Let’s look at some examples.

            +

            + +

            +

            If we compress a note too much, in other words, we make a summary that is too brief, we lose the context and it loses all meaning. In the note above, for example, the information it contains is highly discoverable — I can get the gist of it with just a glance.

            +

            But if I come across this note a year from now, I’ll have no idea what it means or why it’s important. It’s too compressed.

            +

            But we can go too far in the opposite direction too. If we make something totally understandable, in other words, if we include every little detail and bit of context, it loses its discoverability.

            +

            + +

            +

            The example above is my notes on the task management software Jira. It has lots of context, making it highly understandable. But it’s not discoverable at all. It would probably take me a couple hours and tremendous mental effort to read through this note and remember enough context to decide whether or not it’s useful. The main points and key insights are hidden somewhere in the noise.

            +

            Getting the balance between compression and context right is not a trivial matter. When the time comes for Future You to decide whether or not to review this note, seconds count. Because Future You will likely be looking for a solution to a problem, not casual reading, they will be making snap decisions on a tight timeline. Faced with a wall of text of questionable value, they are unlikely to take the risk of committing time for review.

            +

            This means that all the summarizing work your Past Self did on this note is wasted. It didn’t pay off back then, and it doesn’t pay off in the future. You successfully sent a packet of information forward through time, but not in a state where it could survive the journey.

            +

            Opportunistic compression

            +

            I’ve found that most people do just fine on the context side of the equation. We know how to take exhaustive notes on a book, a presentation, or a class.

            +

            Progressive Summarization focuses therefore on rebalancing the equation. It is a method for opportunistic compression — summarizing and condensing a piece of information in small spurts, spread across time, in the course of other work, and only doing as much or as little as the information deserves.

            +

            If you remember, compression is a means to improving discoverability. So our design challenge when creating a note is:

            +
            +

            “How do I make what I’m consuming right now easily discoverable for my future self?”

            +
            +

            This isn’t an easy question to answer, because you have no idea what Future You remembers, is interested in, or is working on. You have to summarize the note without knowing what it will be used for. It is general purpose summarization, a much greater challenge than extracting takeaways for just one specific project.

            +

            Progressive Summarization works in “layers” of summarization. Layer 0 is the original, full-length source text.

            +

            Layer 1 is the content that I initially bring into my note-taking program. I don’t have an explicit set of criteria on what to keep. I just capture anything that feels insightful, interesting, or useful.

            +

            This can include virtually any type of media, but for this article I will focus on text. There are many ways of doing this:

            +
              +
            • Copy a paragraph of text from a PDF I’m reading, and paste it into the Evernote menu bar helper +
              • +
            • Type my random thoughts into a new note on the Evernote mobile app +
              • +
            • Dropping a Word document onto the Evernote icon in the dock on my Mac, which adds it to a note as an attachment
              • +
            • Downloading all my Kindle highlights from a book using Bookcision, and then copying and pasting them into a new note
              • +
            • Forward an email with useful information to my personal import address, which automatically imports the whole email to a note
              • +
            • Highlight the best passages of an online article using the web highlighter Liner, which exports directly to Evernote
              • +
            +

            The examples above are from my recommended program Evernote (iOS, Android, Mac, Windows, browsers), but all the major note-taking platforms support the above functionality in one way or another: Bear (Mac and iOS), Simplenote (iOS, Android, Mac, Windows, Linux), Microsoft OneNote (iOS, Android, Mac, Windows), and Google Keep (browsers, iOS, Android).

            +

            Layer 1 is the starting point of Progressive Summarization, like the bedrock on which everything else is built:

            +

            + +

            +

            Layer 2 is the first round of true summarization, in which I bold only the best parts of the passages I’ve imported. Again, I have no explicit criteria. I look for keywords, key phrases, and key sentences that I feel represent the core or essence of the idea being discussed.

            +

            + +

            +

            I do this bolding layer at a later time, when I’m already reviewing this note anyway. I’m essentially using the attention I’m already spending for a dual purpose: to “buy” the information I need for the project at hand, and also to summarize the note for future use. If you have to pay attention to something, it comes in handy to be able to double-spend.

            +

            For Layer 3, I switch to highlighting, so I can make out the smaller number of highlighted passages among all the bolded ones. This time, I’m looking for the “best of the best,” only highlighting something if it is truly unique or valuable. And again, I’m only adding this third layer when I’m already reviewing the note anyway.

            +

            + +

            +

            For Layer 4, I’m still summarizing, but going beyond highlighting the words of others, to recording my own. For a small number of notes that are the most insightful, I summarize layers 2 and 3 in an informal executive summary at the top of the note, restating the key points in my own words.

            +

            + +

            +

            Note that all the previous layers are preserved in context, giving you the freedom to leave things out without worrying that you’ll lose them. Summarization is risky — you may be making the wrong decision about what’s important. But with the safety net of multiple layers of preserved notes, you can strike out decisively on daring intellectual expeditions.

            +

            And finally, for a tiny minority of sources, the ones that are so powerful and exciting I want them to become part of how I think and work immediately, I remix them. After pulling them apart and dissecting them from every angle in layers 1–4, I add my own personality and creativity and turn them into something else.

            +

            + +

            +

            This could include a blog post interpreting, critiquing, or extending the argument an author is making, such as in Strategically Constrained, The Inner Game of Work, and Supersizing the Mind.

            +
            +
            +

            +

            Use Progressive Summarization
            to create easy-to-review notes

            +

            +

            I'll send you my Progressive Summarization Cheat Sheet as a thank you when you subscribe to my free weekly newsletter below.

            +
              +

              Look out for an email from hello@fortelabs.co

              +
              +
              +

              But it doesn’t have to be difficult or time-consuming. It could even be…(gasp) fun! Making a sketch, designing a slide, recording a short video on your phone, and sharing on social media are all forms of wrestling deeply with information.

              +
              +
              +
              The first tweet in a tweetstorm I wrote about the book Toyota Kata +
              +
              +

              + In Part II, we’ll look at some examples of Progressive Summarization in action. +

              +
              +
              +

              Follow us for the latest updates and insights around productivity and Building a Second Brain on Twitter, Facebook, Instagram, LinkedIn, and YouTube. And if you're ready to start building your Second Brain, get the book and learn the proven method to organize your digital life and unlock your creative potential.

              +
              +
              +
              +

              +

              Join the Forte Labs Newsletter

              +
              +

              Every Tuesday, I send over 100,000 subscribers new essays, videos, event invites, and other resources designed to level up your productivity and life. +

              +

              ​

              +

              Join us, and I'll send you my Top 10 Most Popular Articles right away as a thank you.

              +
              +
              +
              +
              +
              +
              + Series Navigation: Progressive Summarization +
              + +
              +
              +
              + + +
              +
              \ No newline at end of file diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/source.html b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/source.html new file mode 100644 index 000000000..44a7a65d5 --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/source.html @@ -0,0 +1,1736 @@ + + + + + + + + + + + + Progressive Summarization: A Practical Technique for Designing Discoverable Notes - Forte Labs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
              +
              +
              +
              +
              + +
              +
              +
              +
              +
              +
              + + + +
              +
              +
              +
              +
              +
              +
              + +
              +
              +
              +
              +
              +
              +
              +
              +
              +

              + Progressive Summarization: A Practical Technique for Designing Discoverable Notes +

              +
              +
              +
              +
              + +
              +
              +
              +
              +
                +
              • + Last modified +
                • +
              • + May 16, 2023 +
                • +
              +
              +
              +
              +
              +

              + Estimated reading time: 12 minutes +

              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              +
              + Series Navigation: Progressive Summarization +
              + + + + + + + +
              + Progressive Summarization II: Examples and Metaphors >> +
              +
              +

              +

              + +

              +

              + Modern digital tools make it easy to “capture” information from a wide variety of sources. We know how to snap a picture, type out some notes, record a video, or scan a document. Getting this content from the outside world into the digital world is trivial. +

              +

              + It’s even easier to get content that is already digital from one app to another. We know how to copy and paste text, save an image from a webpage, archive an email attachment, or import a video file. +

              +

              + What is difficult is not transferring content from place to place, but transferring it through time. +

              +

              + You know what I mean: you read a book, investing hours of mental labor in understanding the ideas it presents. You finish the book with a feeling of triumph that you’ve gained a valuable body of knowledge. +

              +

              + But then what? +

              +

              + You may try to apply the science-based methods the book recommends, only to realize it’s not quite as clear-cut as you thought. You may try to change the way you eat, exercise, communicate, or work, trusting in the power of habits. But then the everyday demands of life come rushing back, and you forget what motivated you in the first place. +

              +

              + At this point, people take different paths. Some give up, labeling all “self-help” books a waste of time. Others decide it’s just a problem of remembering everything they read, and invest in fancy memorization techniques. And many people become “infovores,” force-feeding themselves endless books, articles, and courses, in the hope that something will stick. +

              +

              + I want to suggest an alternative to all the approaches above: what you read is good and useful and very important, you’re just reading it at the wrong time. +

              +

              + You’re reading about time management techniques now, but they will only be useful two years from now, when you become a manager and have much greater demands on your time. +

              +

              + You’re watching YouTube videos on online marketing now, but that knowledge can only be put to use in 9 months, when your new online course gets off the ground. +

              +

              + You’re talking to a prospect about his goals and challenges now, but when you could really use that information is next year, when he is taking bids for a huge new contract. +

              +

              + The challenge of knowledge is not acquiring it. In our digital world, you can acquire almost any knowledge at almost any time. +

              +

              + The challenge is knowing which knowledge is worth acquiring. And then building a system to forward bits of it through time, to the future situation or problem or challenge where it is most applicable, and most needed. +

              +

              + At that future point, when you’re applying that knowledge directly to a real-world challenge, you won’t have to worry about memorizing it, integrating it, or even fully understanding it. You will only have to apply it, and any gaps in your understanding will very quickly reveal themselves. By the time you’re done solving a real problem with it, book knowledge has become experiential knowledge. And experiential knowledge is something you carry with you forever. +

              +

              + This is the job of a “second brain” — an external, integrated digital repository for the things you learn and the resources from which they come. It is a storage and retrieval system, packaging bits of knowledge into discrete packets that can be forwarded to various points in time to be reviewed, utilized, or deleted. +

              +
              +
              +
              +
              +

              + Use Progressive Summarization
              + to create easy-to-review notes +

              +
              +
              +

              + I'll send you my Progressive Summarization Cheat Sheet as a thank you when you subscribe to my free weekly newsletter below. +

              +
              +
                +
                +
                + +
                + +
                +
                +

                + Look out for an email from hello@fortelabs.co +

                +
                +
                +
                +

                +   +

                +

                + In The PARA Method, I described a universal system for organizing any kind of digital information from any source. It is a “good enough” system, maintaining notes according to their actionability (which takes just a moment to determine), instead of their meaning (which is ambiguous and depends on the context). +

                +

                + The four top-level categories of PARA — Projects, Areas, Resources, and Archives — are designed to facilitate this process of forwarding knowledge through time. +

                +
                  +
                • By placing a note in a project folder, you are essentially scheduling it for review on the short time horizon of an individual project +
                  • +
                • Notes in area folders are scheduled for less frequent review, whenever you evaluate that area of your work or life +
                  • +
                • Notes in resource folders stand ready for review if and when you decide to take action on that topic +
                  • +
                • And notes in archive folders are in “cold storage,” available if needed but not scheduled for review at any particular time +
                  • +
                +

                + Note that we have re-created the tickler file, except instead of strict time-based horizons (daily, weekly, monthly, annually), they are scheduled contingently — if X happens, when Y arrives, if I want to do Z, etc. +

                +

                + +

                +

                + Planning in terms of contingencies gives us all the benefits of planning and researching, without locking us into rigid routines. We have the ability to massively accelerate, using our repository of accumulated notes as rocket fuel. But the actual decision of whether or not to accelerate, and critically, in which direction, we leave to our Future Self, who is older and wiser. +

                +

                + PARA answers how these “packets of knowledge” are organized: in discrete notes, sorted into 4 categories according to actionability, and resurfaced using RandomNote. +

                +

                + But now we turn to a more fundamental question: how are these packets made? Once we capture something, how do we structure the note so that it’s easily discoverable and usable in the future? How do we make sure what we’re saving today adds value to future projects, even when we can’t predict or even imagine what those projects might be? +

                +

                + That is the job of Progressive Summarization. +

                +

                + Note-first knowledge management +

                +

                + There are two primary schools of thought on how to organize a note-taking program (or really any body of information, but I’ll use terms specific to note-taking apps): +

                +

                + +

                +

                + Tagging-first approaches argue that there should be no explicit hierarchy of notes, notebooks, and stacks. Notes are envisioned as an ever-changing, virtual matrix of interconnected, free-floating ideas. Because many tags can be applied to one note, there are multiple pathways to discover any given note. Locating notes in specific notebooks and folders is seen as limiting and static. +

                +

                + +

                +

                + Although tags have their uses, I don’t believe they work as a primary organizational system. In my experience, relying on tagging is too fragile and requires too much maintenance, spreading attention too uniformly across all notes whether or not they are truly valuable. The virtual matrix sounds cool and futuristic, but our minds are not made to work well with such abstract concepts — we understand placing one thing in one place intuitively and automatically. +

                +

                + The second conventional approach to organizing notes is notebook-first. This basically translates how we organize things in the physical world — in a series of discrete containers — into the digital world. +

                +

                + +

                +

                + Notebook-first is better than tagging-first, in my opinion, mostly because it stays out of the way. It doesn’t try to automate and encroach upon the deeply intuitive act of making connections and seeing patterns. PARA on its own is a notebook-first system. +

                +

                + But if we stopped there, it would still be woefully inadequate for an economy based on creative output. As the tagging enthusiasts correctly point out, notebooks and folders actually suppress the serendipity and randomness that is at the heart of a creative lifestyle. +

                +

                + I propose a way to break the impasse: a note-first approach. +

                +

                + +

                +

                + I propose we make the design of individual notes the primary factor, instead of tags or notebooks. This has many advantages: +

                +
                  +
                • It works well with any other organizational system, without depending on them (including but not limited to tags and notebooks, if you want to use those) +
                  • +
                • It makes all work you do on your notes value-added, because you’re spending close to 100% of the time engaging directly with the content itself +
                  • +
                • It can more easily survive migrations to other devices, storage locations, and even programs, because note content is much more likely to be preserved than overarching structure +
                  • +
                • It cultivates skills (succinct communication, finding the core of an idea, visual thinking, etc.) that are inherently valuable and highly transferrable to other activities +
                  • +
                • It makes your notes more legible and useful to others (unlike your internal notebook structure, which is only for your use), promoting collaboration and sharing +
                  • +
                +

                + With a note-first approach, your notes become like individual atoms — each with its own unique properties, but ready to be assembled into elements, molecules, and compounds that are far more powerful. +

                +

                + Designing discoverable notes +

                +

                + A note-first approach to knowledge management means we have to think about design. You are, in a very real sense, designing a product for a demanding customer — Future You. +

                +

                + Future You doesn’t necessarily trust that everything Past You put into your notes is valuable. Future You is impatient and skeptical, demanding proof upfront that the time they spend reviewing notes will be worthwhile. You’ve gotta “sell them” on the idea of reviewing a given note, including all the stages any salesperson has to master: gaining attention, inspiring interest, establishing credibility, stoking desire, and making a case for action NOW. +

                +

                + As if all that wasn’t intimidating enough, you have to do this for every single note without spending any extra time. You don’t have extra time, do you? +

                +

                + Let’s start at the beginning: at the heart of every design, we are trying to balance priorities. You want one thing, but it has to be balanced against something else that you also want. +

                +

                + You want a vehicle to protect its occupants, but you can’t just add layers and layers of titanium armor plating. You have to balance safety against weight and cost. +

                +

                + You want a phone to have the longest possible battery life, but you can’t just give it a 10-pound brick of a battery. You have to balance battery life against size and usability. +

                +

                + In the case of notes, I believe the two priorities we are trying to balance are discoverability and understanding. +

                +

                + +

                +

                + Making a note discoverable involves making it small, simple, and easy to digest. We accomplish this using compression: creating highly condensed summaries, without all the fluff. +

                +

                + But we also want to make our notes understandable. This involves including all the context: the details, the examples, and cited sources to be sure nothing falls through the cracks. +

                +

                + +

                +

                + This is a difficult tradeoff because you cannot compress something without losing some of its context. +

                +

                + You cannot summarize an article without discarding most of its points. You cannot make a highlight reel of a video without cutting out most of the footage. You cannot give an 18-minute TED talk without leaving out most of your ideas. +

                +

                + In making decisions about what to keep, you are inevitably making decisions about what to throw away. +

                +

                + Compression vs. context +

                +

                + There’s a natural tension between the two, compression and context. +

                +

                + +

                +

                + To communicate anything, you have to compress it, like communicating a huge amount of life experience in a wise saying. But in doing so, you lose a lot of the context that made that wisdom valuable in the first place. +

                +

                + Let’s look at some examples. +

                +

                + +

                +

                + If we compress a note too much, in other words, we make a summary that is too brief, we lose the context and it loses all meaning. In the note above, for example, the information it contains is highly discoverable — I can get the gist of it with just a glance. +

                +

                + But if I come across this note a year from now, I’ll have no idea what it means or why it’s important. It’s too compressed. +

                +

                + But we can go too far in the opposite direction too. If we make something totally understandable, in other words, if we include every little detail and bit of context, it loses its discoverability. +

                +

                + +

                +

                + The example above is my notes on the task management software Jira. It has lots of context, making it highly understandable. But it’s not discoverable at all. It would probably take me a couple hours and tremendous mental effort to read through this note and remember enough context to decide whether or not it’s useful. The main points and key insights are hidden somewhere in the noise. +

                +

                + Getting the balance between compression and context right is not a trivial matter. When the time comes for Future You to decide whether or not to review this note, seconds count. Because Future You will likely be looking for a solution to a problem, not casual reading, they will be making snap decisions on a tight timeline. Faced with a wall of text of questionable value, they are unlikely to take the risk of committing time for review. +

                +

                + This means that all the summarizing work your Past Self did on this note is wasted. It didn’t pay off back then, and it doesn’t pay off in the future. You successfully sent a packet of information forward through time, but not in a state where it could survive the journey. +

                +

                + Opportunistic compression +

                +

                + I’ve found that most people do just fine on the context side of the equation. We know how to take exhaustive notes on a book, a presentation, or a class. +

                +

                + Progressive Summarization focuses therefore on rebalancing the equation. It is a method for opportunistic compression — summarizing and condensing a piece of information in small spurts, spread across time, in the course of other work, and only doing as much or as little as the information deserves. +

                +

                + If you remember, compression is a means to improving discoverability. So our design challenge when creating a note is: +

                +
                +

                + “How do I make what I’m consuming right now easily discoverable for my future self?” +

                +
                +

                + This isn’t an easy question to answer, because you have no idea what Future You remembers, is interested in, or is working on. You have to summarize the note without knowing what it will be used for. It is general purpose summarization, a much greater challenge than extracting takeaways for just one specific project. +

                +

                + Progressive Summarization works in “layers” of summarization. Layer 0 is the original, full-length source text. +

                +

                + Layer 1 is the content that I initially bring into my note-taking program. I don’t have an explicit set of criteria on what to keep. I just capture anything that feels insightful, interesting, or useful. +

                +

                + This can include virtually any type of media, but for this article I will focus on text. There are many ways of doing this: +

                +
                  +
                • Copy a paragraph of text from a PDF I’m reading, and paste it into the Evernote menu bar helper +
                  • +
                • Type my random thoughts into a new note on the Evernote mobile app +
                  • +
                • Dropping a Word document onto the Evernote icon in the dock on my Mac, which adds it to a note as an attachment +
                  • +
                • Downloading all my Kindle highlights from a book using Bookcision, and then copying and pasting them into a new note +
                  • +
                • Forward an email with useful information to my personal import address, which automatically imports the whole email to a note +
                  • +
                • Highlight the best passages of an online article using the web highlighter Liner, which exports directly to Evernote +
                  • +
                +

                + The examples above are from my recommended program Evernote (iOS, Android, Mac, Windows, browsers), but all the major note-taking platforms support the above functionality in one way or another: Bear (Mac and iOS), Simplenote (iOS, Android, Mac, Windows, Linux), Microsoft OneNote (iOS, Android, Mac, Windows), and Google Keep (browsers, iOS, Android). +

                +

                + Layer 1 is the starting point of Progressive Summarization, like the bedrock on which everything else is built: +

                +

                + +

                +

                + Layer 2 is the first round of true summarization, in which I bold only the best parts of the passages I’ve imported. Again, I have no explicit criteria. I look for keywords, key phrases, and key sentences that I feel represent the core or essence of the idea being discussed. +

                +

                + +

                +

                + I do this bolding layer at a later time, when I’m already reviewing this note anyway. I’m essentially using the attention I’m already spending for a dual purpose: to “buy” the information I need for the project at hand, and also to summarize the note for future use. If you have to pay attention to something, it comes in handy to be able to double-spend. +

                +

                + For Layer 3, I switch to highlighting, so I can make out the smaller number of highlighted passages among all the bolded ones. This time, I’m looking for the “best of the best,” only highlighting something if it is truly unique or valuable. And again, I’m only adding this third layer when I’m already reviewing the note anyway. +

                +

                + +

                +

                + For Layer 4, I’m still summarizing, but going beyond highlighting the words of others, to recording my own. For a small number of notes that are the most insightful, I summarize layers 2 and 3 in an informal executive summary at the top of the note, restating the key points in my own words. +

                +

                + +

                +

                + Note that all the previous layers are preserved in context, giving you the freedom to leave things out without worrying that you’ll lose them. Summarization is risky — you may be making the wrong decision about what’s important. But with the safety net of multiple layers of preserved notes, you can strike out decisively on daring intellectual expeditions. +

                +

                + And finally, for a tiny minority of sources, the ones that are so powerful and exciting I want them to become part of how I think and work immediately, I remix them. After pulling them apart and dissecting them from every angle in layers 1–4, I add my own personality and creativity and turn them into something else. +

                +

                + +

                +

                + This could include a blog post interpreting, critiquing, or extending the argument an author is making, such as in Strategically Constrained, The Inner Game of Work, and Supersizing the Mind. +

                +
                +
                +
                +
                +

                + Use Progressive Summarization
                + to create easy-to-review notes +

                +
                +
                +

                + I'll send you my Progressive Summarization Cheat Sheet as a thank you when you subscribe to my free weekly newsletter below. +

                +
                +
                  +
                  +
                  + +
                  + +
                  +
                  +

                  + Look out for an email from hello@fortelabs.co +

                  +
                  +
                  +
                  +

                  +   +

                  +

                  + But it doesn’t have to be difficult or time-consuming. It could even be…(gasp) fun! Making a sketch, designing a slide, recording a short video on your phone, and sharing on social media are all forms of wrestling deeply with information. +

                  + +

                  + +

                  +
                  +
                  + The first tweet in a tweetstorm I wrote about the book Toyota Kata +
                  +
                  +

                  + In Part II, we’ll look at some examples of Progressive Summarization in action. +

                  +
                  +

                  + Follow us for the latest updates and insights around productivity and Building a Second Brain on Twitter, Facebook, Instagram, LinkedIn, and YouTube. And if you're ready to start building your Second Brain, get the book and learn the proven method to organize your digital life and unlock your creative potential.
                  +
                  +


                  +
                  +
                  +
                  +
                  +
                  +

                  + Join the Forte Labs Newsletter +

                  +
                  +
                  +

                  + Every Tuesday, I send over 100,000 subscribers new essays, videos, event invites, and other resources designed to level up your productivity and life. +

                  +

                  + ​ +

                  +

                  + Join us, and I'll send you my Top 10 Most Popular Articles right away as a thank you. +

                  +
                  +
                  +
                  +
                    +
                    +
                    + +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    + Series Navigation: Progressive Summarization +
                    + + + + + + + +
                    + Progressive Summarization II: Examples and Metaphors >> +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +

                    + Level up your productivity & your life with one email per week +

                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +

                    + Join 100,000 people receiving my best ideas on productivity & knowledge management every Tuesday. +

                    +

                    + I’ll send you my Top 10 All-Time Articles right away as a thank you. +

                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    + 5318 E. 2nd St. PMB 36973
                    + Long Beach, CA 90803, USA +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + RESOURCES +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + WORK WITH US +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + GOT QUESTIONS? +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +

                    + SPEAKING INQUIRIES +

                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + CONNECT +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + Twitter Facebook Youtube Linkedin Instagram +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    +
                    + + +
                    +
                      +
                    • + +
                      • +
                    • + +
                      • +
                    • + +
                      • +
                    • + +
                      • +
                    • + +
                      • +
                    • + +
                      • +
                    +
                    +
                    +
                    +
                    + Share This +
                    + +
                    +
                    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
                    + word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word word +

                    + mmMwWLliI0fiflO&1
                    + mmMwWLliI0fiflO&1
                    + mmMwWLliI0fiflO&1
                    + mmMwWLliI0fiflO&1
                    + mmMwWLliI0fiflO&1
                    + mmMwWLliI0fiflO&1
                    + mmMwWLliI0fiflO&1 + + + +
                    +
                    +
                    +
                    + +
                    +
                    +
                    +
                    + + diff --git a/packages/readabilityjs/test/test-pages/newsletters/forte-labs/url.txt b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/url.txt new file mode 100644 index 000000000..1cbb17cab --- /dev/null +++ b/packages/readabilityjs/test/test-pages/newsletters/forte-labs/url.txt @@ -0,0 +1 @@ +https://fortelabs.com/blog/progressive-summarization-a-practical-technique-for-designing-discoverable-notes/ \ No newline at end of file diff --git a/packages/rss-handler/package.json b/packages/rss-handler/package.json index 2a6d40851..0db5d93fd 100644 --- a/packages/rss-handler/package.json +++ b/packages/rss-handler/package.json @@ -17,7 +17,8 @@ "devDependencies": { "chai": "^4.3.6", "eslint-plugin-prettier": "^4.0.0", - "mocha": "^10.0.0" + "mocha": "^10.0.0", + "nock": "^13.3.4" }, "dependencies": { "@google-cloud/functions-framework": "3.1.2", diff --git a/packages/rss-handler/src/index.ts b/packages/rss-handler/src/index.ts index 9c37e22f9..e3b8d896d 100644 --- a/packages/rss-handler/src/index.ts +++ b/packages/rss-handler/src/index.ts @@ -1,5 +1,6 @@ import * as Sentry from '@sentry/serverless' import axios from 'axios' +import crypto from 'crypto' import * as dotenv from 'dotenv' // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import import * as jwt from 'jsonwebtoken' import Parser, { Item } from 'rss-parser' @@ -7,9 +8,12 @@ import { promisify } from 'util' import { CONTENT_FETCH_URL, createCloudTask } from './task' interface RssFeedRequest { - subscriptionId: string + subscriptionIds: string[] feedUrl: string - lastFetchedAt: number // unix timestamp in milliseconds + lastFetchedTimestamps: number[] // unix timestamp in milliseconds + scheduledTimestamps: number[] // unix timestamp in milliseconds + lastFetchedChecksums: string[] + userIds: string[] } // link can be a string or an object @@ -17,14 +21,47 @@ type RssFeedItemLink = string | { $: { rel?: string; href: string } } function isRssFeedRequest(body: any): body is RssFeedRequest { return ( - 'subscriptionId' in body && 'feedUrl' in body && 'lastFetchedAt' in body + 'subscriptionIds' in body && + 'feedUrl' in body && + 'lastFetchedTimestamps' in body && + 'scheduledTimestamps' in body && + 'userIds' in body && + 'lastFetchedChecksums' in body ) } +export const fetchAndChecksum = async (url: string) => { + try { + const response = await axios.get(url, { + responseType: 'arraybuffer', + timeout: 60_000, + maxRedirects: 10, + headers: { + 'User-Agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', + Accept: + 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', + }, + }) + + const hash = crypto.createHash('sha256') + hash.update(response.data as Buffer) + + const dataStr = (response.data as Buffer).toString() + + return { url, content: dataStr, checksum: hash.digest('hex') } + } catch (error) { + console.log(error) + throw new Error(`Failed to fetch or hash content from ${url}.`) + } +} + const sendUpdateSubscriptionMutation = async ( userId: string, subscriptionId: string, - lastFetchedAt: Date + lastFetchedAt: Date, + lastFetchedChecksum: string, + scheduledAt: Date ) => { const JWT_SECRET = process.env.JWT_SECRET const REST_BACKEND_ENDPOINT = process.env.REST_BACKEND_ENDPOINT @@ -51,6 +88,8 @@ const sendUpdateSubscriptionMutation = async ( input: { id: subscriptionId, lastFetchedAt, + lastFetchedChecksum, + scheduledAt, }, }, }) @@ -118,20 +157,63 @@ Sentry.GCPFunction.init({ const signToken = promisify(jwt.sign) const parser = new Parser({ - timeout: 60000, // 60 seconds - maxRedirects: 10, customFields: { - item: [['link', 'links', { keepArray: true }], 'published', 'updated'], - }, - headers: { - // some rss feeds require user agent - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36', - Accept: - 'application/rss+xml, application/rdf+xml;q=0.8, application/atom+xml;q=0.6, application/xml;q=0.4, text/xml;q=0.4', + item: [ + ['link', 'links', { keepArray: true }], + 'published', + 'updated', + 'created', + ], + feed: [ + 'dc:date', + 'lastBuildDate', + 'pubDate', + 'syn:updatePeriod', + 'syn:updateFrequency', + 'sy:updatePeriod', + 'sy:updateFrequency', + ], }, }) +const getUpdateFrequency = (feed: any) => { + const updateFrequency = (feed['syn:updateFrequency'] || + feed['sy:updateFrequency']) as string | undefined + + if (!updateFrequency) { + return 1 + } + + const frequency = parseInt(updateFrequency, 10) + if (isNaN(frequency)) { + return 1 + } + + return frequency +} + +const getUpdatePeriodInHours = (feed: any) => { + const updatePeriod = (feed['syn:updatePeriod'] || feed['sy:updatePeriod']) as + | string + | undefined + + switch (updatePeriod) { + case 'hourly': + return 1 + case 'daily': + return 24 + case 'weekly': + return 7 * 24 + case 'monthly': + return 30 * 24 + case 'yearly': + return 365 * 24 + default: + // default to hourly + return 1 + } +} + // get link following the order of preference: via, alternate, self const getLink = (links: RssFeedItemLink[]) => { // sort links by preference @@ -158,141 +240,176 @@ const getLink = (links: RssFeedItemLink[]) => { return sortedLinks.find((link) => !!link) } -export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( - async (req, res) => { - if (!process.env.JWT_SECRET) { - console.error('Missing JWT_SECRET in environment') - return res.status(500).send('INTERNAL_SERVER_ERROR') +const processSubscription = async ( + subscriptionId: string, + userId: string, + feedUrl: string, + fetchResult: { content: string; checksum: string }, + lastFetchedAt: number, + scheduledAt: number, + lastFetchedChecksum: string +) => { + let lastItemFetchedAt: Date | null = null + let lastValidItem: Item | null = null + + if (fetchResult.checksum === lastFetchedChecksum) { + console.log('feed has not been updated', feedUrl, lastFetchedChecksum) + return + } + const updatedLastFetchedChecksum = fetchResult.checksum + + // fetch feed + let itemCount = 0 + const feed = await parser.parseString(fetchResult.content) + console.log('Fetched feed', feed.title, new Date()) + + const feedPubDate = (feed['dc:date'] || + feed.pubDate || + feed.lastBuildDate) as string | undefined + console.log('Feed pub date', feedPubDate) + if (feedPubDate && new Date(feedPubDate) < new Date(lastFetchedAt)) { + console.log('Skipping old feed', feedPubDate) + return + } + + // save each item in the feed + for (const item of feed.items) { + // use published or updated if isoDate is not available for atom feeds + item.isoDate = + item.isoDate || + (item.published as string) || + (item.updated as string) || + (item.created as string) + console.log('Processing feed item', item.links, item.isoDate) + + if (!item.links || item.links.length === 0) { + console.log('Invalid feed item', item) + continue } - const token = req.header('Omnivore-Authorization') - if (!token) { - console.error('Missing authorization header') - return res.status(401).send('UNAUTHORIZED') + item.link = getLink(item.links as RssFeedItemLink[]) + if (!item.link) { + console.log('Invalid feed item links', item.links) + continue + } + + console.log('Fetching feed item', item.link) + + const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() + // remember the last valid item + if ( + !lastValidItem || + (lastValidItem.isoDate && publishedAt > new Date(lastValidItem.isoDate)) + ) { + lastValidItem = item + } + + // Max limit per-feed update + if (itemCount > 99) { + continue + } + + // skip old items and items that were published before 24h + if ( + publishedAt < new Date(lastFetchedAt) || + publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) + ) { + console.log('Skipping old feed item', item.link) + continue + } + + const created = await createSavingItemTask(userId, feedUrl, item) + if (!created) { + console.error('Failed to create task for feed item', item.link) + continue + } + + // remember the last item fetched at + if (!lastItemFetchedAt || publishedAt > lastItemFetchedAt) { + lastItemFetchedAt = publishedAt + } + + itemCount = itemCount + 1 + } + + // no items saved + if (!lastItemFetchedAt) { + // the feed has been fetched before, no new valid items found + if (lastFetchedAt || !lastValidItem) { + console.log('No new valid items found') + return + } + + // the feed has never been fetched, save at least the last valid item + const created = await createSavingItemTask(userId, feedUrl, lastValidItem) + if (!created) { + console.error('Failed to create task for feed item', lastValidItem.link) + throw new Error('Failed to create task for feed item') + } + + lastItemFetchedAt = lastValidItem.isoDate + ? new Date(lastValidItem.isoDate) + : new Date() + } + + const updateFrequency = getUpdateFrequency(feed) + const updatePeriodInMs = getUpdatePeriodInHours(feed) * 60 * 60 * 1000 + const nextScheduledAt = scheduledAt + updatePeriodInMs * updateFrequency + + // update subscription lastFetchedAt + const updatedSubscription = await sendUpdateSubscriptionMutation( + userId, + subscriptionId, + lastItemFetchedAt, + updatedLastFetchedChecksum, + new Date(nextScheduledAt) + ) + console.log('Updated subscription', updatedSubscription) +} + +export const rssHandler = Sentry.GCPFunction.wrapHttpFunction( + async (req, res) => { + if (req.query.token !== process.env.PUBSUB_VERIFICATION_TOKEN) { + console.log('query does not include valid token') + return res.sendStatus(403) } try { - let userId: string - - try { - const decoded = jwt.verify(token, process.env.JWT_SECRET) as { - uid: string - } - userId = decoded.uid - } catch (e) { - console.error('Authorization error', e) - return res.status(401).send('UNAUTHORIZED') - } - if (!isRssFeedRequest(req.body)) { console.error('Invalid request body', req.body) return res.status(400).send('INVALID_REQUEST_BODY') } - const { feedUrl, subscriptionId, lastFetchedAt } = req.body - console.log('Processing feed', feedUrl, lastFetchedAt) + const { + feedUrl, + subscriptionIds, + lastFetchedTimestamps, + scheduledTimestamps, + userIds, + lastFetchedChecksums, + } = req.body + console.log('Processing feed', feedUrl) - let lastItemFetchedAt: Date | null = null - let lastValidItem: Item | null = null + const fetchResult = await fetchAndChecksum(feedUrl) - // fetch feed - let itemCount = 0 - const feed = await parser.parseURL(feedUrl) - console.log('Fetched feed', feed.title, new Date()) + for (let i = 0; i < subscriptionIds.length; i++) { + const subscriptionId = subscriptionIds[i] + const lastFetchedAt = lastFetchedTimestamps[i] + const scheduledAt = scheduledTimestamps[i] + const userId = userIds[i] + const lastFetchedChecksum = lastFetchedChecksums[i] - // save each item in the feed - for (const item of feed.items) { - // use published or updated if isoDate is not available for atom feeds - item.isoDate = - item.isoDate || (item.published as string) || (item.updated as string) - console.log('Processing feed item', item.links, item.isoDate) - - if (!item.links || item.links.length === 0) { - console.log('Invalid feed item', item) - continue - } - - item.link = getLink(item.links as RssFeedItemLink[]) - if (!item.link) { - console.log('Invalid feed item links', item.links) - continue - } - - console.log('Fetching feed item', item.link) - - const publishedAt = item.isoDate ? new Date(item.isoDate) : new Date() - // remember the last valid item - if ( - !lastValidItem || - (lastValidItem.isoDate && - publishedAt > new Date(lastValidItem.isoDate)) - ) { - lastValidItem = item - } - - // Max limit per-feed update - if (itemCount > 99) { - continue - } - - // skip old items and items that were published before 24h - if ( - publishedAt < new Date(lastFetchedAt) || - publishedAt < new Date(Date.now() - 24 * 60 * 60 * 1000) - ) { - console.log('Skipping old feed item', item.link) - continue - } - - const created = await createSavingItemTask(userId, feedUrl, item) - if (!created) { - console.error('Failed to create task for feed item', item.link) - continue - } - - // remember the last item fetched at - if (!lastItemFetchedAt || publishedAt > lastItemFetchedAt) { - lastItemFetchedAt = publishedAt - } - - itemCount = itemCount + 1 - } - - // no items saved - if (!lastItemFetchedAt) { - // the feed has been fetched before, no new valid items found - if (lastFetchedAt || !lastValidItem) { - console.log('No new valid items found') - return res.send('ok') - } - - // the feed has never been fetched, save at least the last valid item - const created = await createSavingItemTask( + await processSubscription( + subscriptionId, userId, feedUrl, - lastValidItem + fetchResult, + lastFetchedAt, + scheduledAt, + lastFetchedChecksum ) - if (!created) { - console.error( - 'Failed to create task for feed item', - lastValidItem.link - ) - return res.status(500).send('INTERNAL_SERVER_ERROR') - } - - lastItemFetchedAt = lastValidItem.isoDate - ? new Date(lastValidItem.isoDate) - : new Date() } - // update subscription lastFetchedAt - const updatedSubscription = await sendUpdateSubscriptionMutation( - userId, - subscriptionId, - lastItemFetchedAt - ) - console.log('Updated subscription', updatedSubscription) - res.send('ok') } catch (e) { console.error('Error while parsing RSS feed', e) diff --git a/packages/rss-handler/test/checksum.test.ts b/packages/rss-handler/test/checksum.test.ts new file mode 100644 index 000000000..24eee7a0b --- /dev/null +++ b/packages/rss-handler/test/checksum.test.ts @@ -0,0 +1,14 @@ +import 'mocha' +import nock from 'nock' +import { expect } from 'chai' +import { fetchAndChecksum } from '../src/index' + +describe('fetchAndChecksum', () => { + it('should hash the content available', async () => { + nock('https://fake.com', {}).get('/rss.xml').reply(200, 'i am some content') + const result = await fetchAndChecksum('https://fake.com/rss.xml') + expect(result.checksum).to.eq( + 'd6bc10faec048d999d0cf4b2f7103d84557fb9cd94c3bccd17884b1288949375' + ) + }) +}) diff --git a/packages/rule-handler/src/filter.ts b/packages/rule-handler/src/filter.ts index a3c43e902..2eb365c05 100644 --- a/packages/rule-handler/src/filter.ts +++ b/packages/rule-handler/src/filter.ts @@ -6,6 +6,9 @@ interface SearchResponse { edges: Edge[] } } + errors?: { + message: string + }[] } interface Edge { @@ -28,7 +31,6 @@ interface Label { } export const search = async ( - userId: string, apiEndpoint: string, auth: string, query: string @@ -75,6 +77,12 @@ export const search = async ( } ) + if (response.data.errors) { + console.error(response.data.errors) + + return [] + } + const edges = response.data.data.search.edges if (edges.length === 0) { return [] @@ -89,14 +97,13 @@ export const search = async ( } export const filterPage = async ( - userId: string, apiEndpoint: string, auth: string, filter: string, pageId: string ): Promise => { filter += ` includes:${pageId}` - const pages = await search(userId, apiEndpoint, auth, filter) + const pages = await search(apiEndpoint, auth, filter) return pages.length > 0 ? pages[0] : null } diff --git a/packages/rule-handler/src/rule.ts b/packages/rule-handler/src/rule.ts index b7345fb2c..ef80aa973 100644 --- a/packages/rule-handler/src/rule.ts +++ b/packages/rule-handler/src/rule.ts @@ -93,7 +93,6 @@ export const triggerActions = async ( } const filteredPage = await filterPage( - userId, apiEndpoint, authToken, rule.filter, diff --git a/packages/web/components/elements/icons/FavoriteFlairIcon.tsx b/packages/web/components/elements/icons/FavoriteFlairIcon.tsx new file mode 100644 index 000000000..ab7ab8386 --- /dev/null +++ b/packages/web/components/elements/icons/FavoriteFlairIcon.tsx @@ -0,0 +1,29 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class FavoriteFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + ) + } +} diff --git a/packages/web/components/elements/icons/FeedFlairIcon.tsx b/packages/web/components/elements/icons/FeedFlairIcon.tsx new file mode 100644 index 000000000..1b1c4b78f --- /dev/null +++ b/packages/web/components/elements/icons/FeedFlairIcon.tsx @@ -0,0 +1,44 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class FeedFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + + + ) + } +} diff --git a/packages/web/components/elements/icons/NewsletterFlairIcon.tsx b/packages/web/components/elements/icons/NewsletterFlairIcon.tsx new file mode 100644 index 000000000..47cef57f8 --- /dev/null +++ b/packages/web/components/elements/icons/NewsletterFlairIcon.tsx @@ -0,0 +1,33 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class NewsletterFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + + ) + } +} diff --git a/packages/web/components/elements/icons/PinnedFlairIcon.tsx b/packages/web/components/elements/icons/PinnedFlairIcon.tsx new file mode 100644 index 000000000..f563a93bb --- /dev/null +++ b/packages/web/components/elements/icons/PinnedFlairIcon.tsx @@ -0,0 +1,29 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class PinnedFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + ) + } +} diff --git a/packages/web/components/elements/icons/RecommendedFlairIcon.tsx b/packages/web/components/elements/icons/RecommendedFlairIcon.tsx new file mode 100644 index 000000000..aec665505 --- /dev/null +++ b/packages/web/components/elements/icons/RecommendedFlairIcon.tsx @@ -0,0 +1,33 @@ +/* eslint-disable functional/no-class */ +/* eslint-disable functional/no-this-expression */ +import { IconProps } from './IconProps' + +import React from 'react' + +export class RecommendedFlairIcon extends React.Component { + render() { + const size = (this.props.size || 26).toString() + const color = (this.props.color || '#2A2A2A').toString() + + return ( + + + + + + + ) + } +} diff --git a/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx b/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx index 52e4ddd43..a27570122 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryCardStyles.tsx @@ -2,7 +2,13 @@ import dayjs from 'dayjs' import relativeTime from 'dayjs/plugin/relativeTime' import { ChangeEvent, useMemo } from 'react' import { LibraryItemNode } from '../../../lib/networking/queries/useGetLibraryItemsQuery' -import { Box, SpanBox, VStack } from '../../elements/LayoutPrimitives' +import { Box, HStack, SpanBox, VStack } from '../../elements/LayoutPrimitives' +import { RecommendedFlairIcon } from '../../elements/icons/RecommendedFlairIcon' +import { PinnedFlairIcon } from '../../elements/icons/PinnedFlairIcon' +import { FavoriteFlairIcon } from '../../elements/icons/FavoriteFlairIcon' +import { NewsletterFlairIcon } from '../../elements/icons/NewsletterFlairIcon' +import { FeedFlairIcon } from '../../elements/icons/FeedFlairIcon' +import { Label } from '../../../lib/networking/fragments/labelFragment' dayjs.extend(relativeTime) @@ -83,6 +89,52 @@ const shouldHideUrl = (url: string): boolean => { return false } +export const FLAIR_ICON_NAMES = [ + 'favorite', + 'pinned', + 'recommended', + 'newsletter', + 'feed', + 'rss', +] + +const flairIconForLabel = (label: Label): JSX.Element | undefined => { + switch (label.name.toLocaleLowerCase()) { + case 'favorite': + return ( + + + + ) + case 'pinned': + return ( + + + + ) + case 'recommended': + return ( + + + + ) + case 'newsletter': + return ( + + + + ) + case 'rss': + case 'feed': + return ( + + + + ) + } + return undefined +} + export const siteName = ( originalArticleUrl: string, itemUrl: string @@ -114,7 +166,10 @@ export function LibraryItemMetadata( }, [props.item.highlights]) return ( - + + {props.item.labels?.map((label) => { + return flairIconForLabel(label) + })} {timeAgo(props.item.savedAt)} {` `} {props.item.wordsCount ?? 0 > 0 @@ -126,7 +181,7 @@ export function LibraryItemMetadata( {highlightCount > 0 ? ` • ${highlightCount} highlight${highlightCount > 1 ? 's' : ''}` : null} - + ) } diff --git a/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx b/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx index 0ba2db164..0d139fbd3 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx @@ -13,6 +13,7 @@ import { siteName, TitleStyle, MenuStyle, + FLAIR_ICON_NAMES, } from './LibraryCardStyles' import { sortedLabels } from '../../../lib/labelsSort' import { LibraryHoverActions } from './LibraryHoverActions' @@ -285,9 +286,14 @@ const LibraryGridCardContent = (props: LinkedItemCardProps): JSX.Element => { marginLeft: '-4px', // offset because the chips have margin }} > - {sortedLabels(props.item.labels).map(({ name, color }, index) => ( - - ))} + {sortedLabels(props.item.labels) + .filter( + ({ name }) => + FLAIR_ICON_NAMES.indexOf(name.toLocaleLowerCase()) == -1 + ) + .map(({ name, color }, index) => ( + + ))} diff --git a/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx b/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx index 95b8c4002..666a0b04c 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx @@ -11,6 +11,7 @@ import { siteName, TitleStyle, MenuStyle, + FLAIR_ICON_NAMES, } from './LibraryCardStyles' import { sortedLabels } from '../../../lib/labelsSort' import { LIBRARY_LEFT_MENU_WIDTH } from '../../templates/homeFeed/LibraryFilterMenu' @@ -354,9 +355,14 @@ export function LibraryListCardContent( display: 'block', }} > - {sortedLabels(props.item.labels).map(({ name, color }, index) => ( - - ))} + {sortedLabels(props.item.labels) + .filter( + ({ name }) => + FLAIR_ICON_NAMES.indexOf(name.toLocaleLowerCase()) == -1 + ) + .map(({ name, color }, index) => ( + + ))} diff --git a/packages/web/components/templates/article/PdfArticleContainer.tsx b/packages/web/components/templates/article/PdfArticleContainer.tsx index 210b1b2ae..4c8ba5bbd 100644 --- a/packages/web/components/templates/article/PdfArticleContainer.tsx +++ b/packages/web/components/templates/article/PdfArticleContainer.tsx @@ -408,11 +408,9 @@ export default function PdfArticleContainer( 100, Math.max(0, ((pageIndex + 1) / instance.totalPageCount) * 100) ) - if (percent <= props.article.readingProgressPercent) { - return - } await articleReadingProgressMutation({ id: props.article.id, + force: true, readingProgressPercent: percent, readingProgressAnchorIndex: pageIndex, }) diff --git a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx index 11adec862..fba2015c9 100644 --- a/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx +++ b/packages/web/components/templates/homeFeed/LibraryFilterMenu.tsx @@ -48,32 +48,39 @@ export function LibraryFilterMenu(props: LibraryFilterMenuProps): JSX.Element { isSessionStorage: false, initialValue: [], }) - const { labels: networkLabels, isLoading: labelsLoading } = - useGetLabelsQuery() - const { savedSearches: networkSearches, isLoading: searchesLoading } = - useGetSavedSearchQuery() - const { - subscriptions: networkSubscriptions, - isLoading: subscriptionsLoading, - } = useGetSubscriptionsQuery() + const labelsResponse = useGetLabelsQuery() + const searchesResponse = useGetSavedSearchQuery() + const subscriptionsResponse = useGetSubscriptionsQuery() useEffect(() => { - if (!labelsLoading) { - setLabels(networkLabels) + if ( + !labelsResponse.error && + !labelsResponse.isLoading && + labelsResponse.labels + ) { + setLabels(labelsResponse.labels) } - }, [setLabels, networkLabels, labelsLoading]) + }, [setLabels, labelsResponse]) useEffect(() => { - if (!subscriptionsLoading) { - setSubscriptions(networkSubscriptions) + if ( + !subscriptionsResponse.error && + !subscriptionsResponse.isLoading && + subscriptionsResponse.subscriptions + ) { + setSubscriptions(subscriptionsResponse.subscriptions) } - }, [setSubscriptions, networkSubscriptions, subscriptionsLoading]) + }, [setSubscriptions, subscriptionsResponse]) useEffect(() => { - if (!searchesLoading) { - setSavedSearches(networkSearches ?? []) + if ( + !searchesResponse.error && + !searchesResponse.isLoading && + searchesResponse.savedSearches + ) { + setSavedSearches(searchesResponse.savedSearches) } - }, [setSavedSearches, networkSearches, searchesLoading]) + }, [setSavedSearches, searchesResponse]) return ( <> diff --git a/packages/web/lib/networking/mutations/articleReadingProgressMutation.ts b/packages/web/lib/networking/mutations/articleReadingProgressMutation.ts index 6c41950d0..95fd754d7 100644 --- a/packages/web/lib/networking/mutations/articleReadingProgressMutation.ts +++ b/packages/web/lib/networking/mutations/articleReadingProgressMutation.ts @@ -3,6 +3,7 @@ import { gqlFetcher } from '../networkHelpers' export type ArticleReadingProgressMutationInput = { id: string + force?: boolean readingProgressPercent?: number readingProgressTopPercent?: number readingProgressAnchorIndex?: number diff --git a/packages/web/lib/networking/queries/useGetLabelsQuery.tsx b/packages/web/lib/networking/queries/useGetLabelsQuery.tsx index d6fefad8e..abd4295fd 100644 --- a/packages/web/lib/networking/queries/useGetLabelsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetLabelsQuery.tsx @@ -4,6 +4,7 @@ import { Label, labelFragment } from '../fragments/labelFragment' import { publicGqlFetcher } from '../networkHelpers' type LabelsQueryResponse = { + error: any isLoading: boolean isValidating: boolean labels: Label[] @@ -38,10 +39,11 @@ export function useGetLabelsQuery(): LabelsQueryResponse { const { data, error, mutate, isValidating } = useSWR(query, publicGqlFetcher) try { - if (data) { + if (data && !error) { const result = data as LabelsResponseData const labels = result.labels?.labels as Label[] return { + error, isLoading: !error && !data, isValidating, labels, @@ -54,6 +56,7 @@ export function useGetLabelsQuery(): LabelsQueryResponse { console.log('error', error) } return { + error, isLoading: !error && !data, isValidating: false, labels: [], diff --git a/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx b/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx index 4dde76f89..f6a76f386 100644 --- a/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx +++ b/packages/web/lib/networking/queries/useGetSavedSearchQuery.tsx @@ -7,6 +7,7 @@ import { } from '../fragments/savedSearchFragment' type SavedSearchResponse = { + error: any savedSearches?: SavedSearch[] savedSearchErrors?: unknown isLoading: boolean @@ -39,6 +40,7 @@ export function useGetSavedSearchQuery(): SavedSearchResponse { const { filters } = data as SavedSearchResponseData return { + error, savedSearches: filters?.filters ?? [], savedSearchErrors: error ?? {}, isLoading: false, @@ -46,6 +48,7 @@ export function useGetSavedSearchQuery(): SavedSearchResponse { } return { + error, savedSearches: [], savedSearchErrors: null, isLoading: !error && !data, diff --git a/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx b/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx index 4c0a26d17..ababa6fdb 100644 --- a/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx +++ b/packages/web/lib/networking/queries/useGetSubscriptionsQuery.tsx @@ -25,6 +25,7 @@ export type Subscription = { } type SubscriptionsQueryResponse = { + error: any isLoading: boolean isValidating: boolean subscriptions: Subscription[] @@ -85,6 +86,7 @@ export function useGetSubscriptionsQuery( const result = data as SubscriptionsResponseData const subscriptions = result.subscriptions.subscriptions as Subscription[] return { + error, isLoading: !error && !data, isValidating, subscriptions, @@ -97,6 +99,7 @@ export function useGetSubscriptionsQuery( console.log('error', error) } return { + error, isLoading: !error && !data, isValidating: true, subscriptions: [], diff --git a/pkg/admin/src/db.ts b/pkg/admin/src/db.ts index 471ddf9bb..15435dfe5 100644 --- a/pkg/admin/src/db.ts +++ b/pkg/admin/src/db.ts @@ -50,6 +50,8 @@ export const registerDatabase = async (secrets: any): Promise => { Group, Integration, Subscription, + LibraryItem, + UploadFile, ], }) @@ -308,3 +310,72 @@ export class Subscription extends BaseEntity { @Column({ type: 'timestamp', name: 'updated_at' }) updatedAt!: Date } + +@Entity({ name: 'library_item' }) +export class LibraryItem extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id!: string + + @JoinColumn({ name: 'user_id' }) + @ManyToOne(() => User, (user) => user.articles, { eager: true }) + user!: User + + @Column({ type: 'text', name: 'original_url' }) + originalUrl!: string + + @Column('text') + slug!: string + + @Column('text') + title!: string + + @Column('text', { nullable: true }) + author?: string | null + + @Column('text', { nullable: true }) + subscription?: string | null + + @OneToOne(() => UploadFile, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'upload_file_id' }) + uploadFile?: UploadFile + + @Column({ type: 'timestamp', name: 'saved_at' }) + savedAt!: Date + + @Column({ type: 'timestamp', name: 'deleted_at' }) + deletedAt?: Date | null + + @Column({ type: 'timestamp', name: 'created_at' }) + createdAt!: Date + + @Column({ type: 'timestamp', name: 'updated_at' }) + updatedAt!: Date +} + +@Entity({ name: 'upload_files' }) +export class UploadFile extends BaseEntity { + @PrimaryGeneratedColumn('uuid') + id!: string + + @JoinColumn({ name: 'user_id' }) + @ManyToOne(() => User, (user) => user.articles, { eager: true }) + user!: User + + @Column('text') + url!: string + + @Column('text') + fileName!: string + + @Column('text') + contentType!: string + + @Column('text') + status!: string + + @Column({ type: 'timestamp', name: 'created_at' }) + createdAt!: Date + + @Column({ type: 'timestamp', name: 'updated_at' }) + updatedAt!: Date +} diff --git a/pkg/admin/src/index.ts b/pkg/admin/src/index.ts index 687ddfb59..543c0d987 100644 --- a/pkg/admin/src/index.ts +++ b/pkg/admin/src/index.ts @@ -12,6 +12,7 @@ import { ContentDisplayReport, Subscription, Integration, + LibraryItem, } from './db' import { compare, hashSync } from 'bcryptjs' const readYamlFile = require('read-yaml-file') @@ -41,6 +42,7 @@ const ADMIN_USER_EMAIL = { resource: Group, options: { parent: { name: 'Users' } } }, { resource: Subscription, options: { parent: { name: 'Users' } } }, { resource: Integration, options: { parent: { name: 'Users' } } }, + { resource: LibraryItem, options: { parent: { name: 'Users' } } }, { resource: ContentDisplayReport, }, diff --git a/pkg/admin/yarn.lock b/pkg/admin/yarn.lock index eb2623a42..0b67d4ca9 100644 --- a/pkg/admin/yarn.lock +++ b/pkg/admin/yarn.lock @@ -33,6 +33,14 @@ dependencies: "@babel/highlight" "^7.14.5" +"@babel/code-frame@^7.22.13": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== + dependencies: + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" + "@babel/compat-data@^7.13.11", "@babel/compat-data@^7.14.5", "@babel/compat-data@^7.14.7": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.7.tgz#7b047d7a3a89a67d2258dc61f604f098f1bc7e08" @@ -68,6 +76,16 @@ jsesc "^2.5.1" source-map "^0.5.0" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-annotate-as-pure@^7.0.0", "@babel/helper-annotate-as-pure@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.14.5.tgz#7bf478ec3b71726d56a8ca5775b046fc29879e61" @@ -127,6 +145,11 @@ resolve "^1.14.2" semver "^6.1.2" +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== + "@babel/helper-explode-assignable-expression@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-explode-assignable-expression/-/helper-explode-assignable-expression-7.14.5.tgz#8aa72e708205c7bb643e45c73b4386cdf2a1f645" @@ -143,6 +166,14 @@ "@babel/template" "^7.14.5" "@babel/types" "^7.14.5" +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== + dependencies: + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" + "@babel/helper-get-function-arity@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.14.5.tgz#25fbfa579b0937eee1f3b805ece4ce398c431815" @@ -157,6 +188,13 @@ dependencies: "@babel/types" "^7.14.5" +"@babel/helper-hoist-variables@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz#c01a007dac05c085914e8fb652b339db50d823bb" + integrity sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw== + dependencies: + "@babel/types" "^7.22.5" + "@babel/helper-member-expression-to-functions@^7.14.5": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.14.7.tgz#97e56244beb94211fe277bd818e3a329c66f7970" @@ -237,11 +275,28 @@ dependencies: "@babel/types" "^7.14.5" +"@babel/helper-split-export-declaration@^7.22.6": + version "7.22.6" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz#322c61b7310c0997fe4c323955667f18fcefb91c" + integrity sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g== + dependencies: + "@babel/types" "^7.22.5" + +"@babel/helper-string-parser@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz#533f36457a25814cf1df6488523ad547d784a99f" + integrity sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw== + "@babel/helper-validator-identifier@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.5.tgz#d0f0e277c512e0c938277faa85a3968c9a44c0e8" integrity sha512-5lsetuxCLilmVGyiLEfoHBRX8UCFD+1m2x3Rj97WrW3V7H3u4RWRXA4evMjImCsin2J2YT0QaVDGf+z8ondbAg== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-option@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.14.5.tgz#6e72a1fff18d5dfcb878e1e62f1a021c4b72d5a3" @@ -275,11 +330,25 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/parser@^7.10.2", "@babel/parser@^7.12.5", "@babel/parser@^7.14.5", "@babel/parser@^7.14.6", "@babel/parser@^7.14.7": +"@babel/highlight@^7.22.13": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== + dependencies: + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" + js-tokens "^4.0.0" + +"@babel/parser@^7.10.2", "@babel/parser@^7.12.5", "@babel/parser@^7.14.5", "@babel/parser@^7.14.6": version "7.14.7" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.7.tgz#6099720c8839ca865a2637e6c85852ead0bdb595" integrity sha512-X67Z5y+VBJuHB/RjwECp8kSl5uYi0BvRbNeWqkaJCVh+LiTPl19WBUfG627psSgp9rSf6ojuXghQM3ha6qHHdA== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining/-/plugin-bugfix-v8-spread-parameters-in-optional-chaining-7.14.5.tgz#4b467302e1548ed3b1be43beae2cc9cf45e0bb7e" @@ -977,18 +1046,28 @@ "@babel/parser" "^7.14.5" "@babel/types" "^7.14.5" -"@babel/traverse@^7.12.5", "@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.4.5": - version "7.14.7" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.7.tgz#64007c9774cfdc3abd23b0780bc18a3ce3631753" - integrity sha512-9vDr5NzHu27wgwejuKL7kIOm4bwEtaPQ4Z6cpCmjSuaRqpH/7xc4qcGEscwMqlkwgcXl6MvqoAjZkQ24uSdIZQ== +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== dependencies: - "@babel/code-frame" "^7.14.5" - "@babel/generator" "^7.14.5" - "@babel/helper-function-name" "^7.14.5" - "@babel/helper-hoist-variables" "^7.14.5" - "@babel/helper-split-export-declaration" "^7.14.5" - "@babel/parser" "^7.14.7" - "@babel/types" "^7.14.5" + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" + +"@babel/traverse@^7.12.5", "@babel/traverse@^7.13.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.4.5": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -1000,6 +1079,15 @@ "@babel/helper-validator-identifier" "^7.14.5" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@carbon/icon-helpers@^10.19.0": version "10.19.0" resolved "https://registry.yarnpkg.com/@carbon/icon-helpers/-/icon-helpers-10.19.0.tgz#f6b608b181b4ca4aeeadac72ec11b7cf530b4d1c" @@ -1131,6 +1219,38 @@ gud "^1.0.0" warning "^4.0.3" +"@jridgewell/gen-mapping@^0.3.2": + version "0.3.3" + resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098" + integrity sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ== + dependencies: + "@jridgewell/set-array" "^1.0.1" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping" "^0.3.9" + +"@jridgewell/resolve-uri@^3.1.0": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" + integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + +"@jridgewell/set-array@^1.0.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" + integrity sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw== + +"@jridgewell/sourcemap-codec@^1.4.10", "@jridgewell/sourcemap-codec@^1.4.14": + version "1.4.15" + resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" + integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== + +"@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9": + version "0.3.19" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.19.tgz#f8a3249862f91be48d3127c3cfe992f79b4b8811" + integrity sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw== + dependencies: + "@jridgewell/resolve-uri" "^3.1.0" + "@jridgewell/sourcemap-codec" "^1.4.14" + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" diff --git a/yarn.lock b/yarn.lock index 3125e7f03..90f5c0f62 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2228,17 +2228,6 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw== -"@elastic/elasticsearch@~7.12.0": - version "7.12.0" - resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.12.0.tgz#dbb51a2841f644b670a56d8c15899e860928856f" - integrity sha512-GquUEytCijFRPEk3DKkkDdyhspB3qbucVQOwih9uNyz3iz804I+nGBUsFo2LwVvLQmQfEM0IY2+yoYfEz5wMug== - dependencies: - debug "^4.3.1" - hpagent "^0.1.1" - ms "^2.1.3" - pump "^3.0.0" - secure-json-parse "^2.3.1" - "@emotion/cache@^10.0.27": version "10.0.29" resolved "https://registry.yarnpkg.com/@emotion/cache/-/cache-10.0.29.tgz#87e7e64f412c060102d589fe7c6dc042e6f9d1e0" @@ -15835,11 +15824,6 @@ hpack.js@^2.1.6: readable-stream "^2.0.1" wbuf "^1.1.0" -hpagent@^0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9" - integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ== - html-encoding-sniffer@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" @@ -20177,7 +20161,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -24468,11 +24452,6 @@ search-query-parser@^1.6.0: resolved "https://registry.yarnpkg.com/search-query-parser/-/search-query-parser-1.6.0.tgz#d69ade33f3685cae25613a70189b7b18970b46f1" integrity sha512-bhf+phLlKF38nuniwLcVHWPArHGdzenlPhPi955CR3vm1QQifXIuPHwAffhjapojdVVzmv4hgIJ6NOX1d/w+Uw== -secure-json-parse@^2.3.1: - version "2.4.0" - resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85" - integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg== - selderee@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7"