Merge highlight improvements

README.md

@@ -19,12 +19,14 @@ We built Omnivore because we love reading and we want it to be more social. Join
 - PDF support
 - [Web app](https://omnivore.app/) written in Node.js and TypeScript
 - [Native iOS app](https://omnivore.app/install/ios)
+- [Android app](https://omnivore.app/install/android) ([source](https://github.com/omnivore-app/omnivore/tree/main/android/Omnivore))
 - Progressive web app for Android users
 - Browser extensions for [Chrome](https://omnivore.app/install/chrome), [Safari](https://omnivore.app/install/safari), [Firefox](https://omnivore.app/install/firefox), and [Edge](https://omnivore.app/install/edge)
 - Labels (aka tagging)
 - Offline support
 - Text to speech (iOS only)
 - [Logseq](https://logseq.com/) support via our [Logseq Plugin](https://github.com/omnivore-app/logseq-omnivore)
+- [Obsidian](https://obsidian.md/) support via our [Obsidian Plugin](https://github.com/omnivore-app/obsidian-omnivore)
 
 Every single part is fully open source! Fork it, extend it, or deploy it to your own server.
 

android/Omnivore/app/build.gradle

@@ -17,8 +17,8 @@ android {
         applicationId "app.omnivore.omnivore"
         minSdk 26
         targetSdk 33
-        versionCode 19
-        versionName "0.0.19"
+        versionCode 26
+        versionName "0.0.26"
 
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         vectorDrawables {

android/Omnivore/app/src/main/java/app/omnivore/omnivore/Constants.kt

@@ -1,10 +1,7 @@
 package app.omnivore.omnivore
 
-import app.omnivore.omnivore.ui.reader.WebFont
-
 object Constants {
   const val apiURL = BuildConfig.OMNIVORE_API_URL
-  const val webURL = BuildConfig.OMNIVORE_WEB_URL
   const val dataStoreName = "omnivore-datastore"
 }
 
@@ -18,6 +15,8 @@ object DatastoreKeys {
   const val preferredWebMaxWidthPercentage = "preferredWebMaxWidthPercentage"
   const val preferredWebFontFamily = "preferredWebFontFamily"
   const val prefersWebHighContrastText = "prefersWebHighContrastText"
+  const val lastUsedSavedItemFilter = "lastUsedSavedItemFilter"
+  const val lastUsedSavedItemSortFilter = "lastUsedSavedItemSortFilter"
 }
 
 object AppleConstants {

android/Omnivore/app/src/main/java/app/omnivore/omnivore/MainActivity.kt

@@ -15,7 +15,6 @@ import androidx.core.view.WindowCompat
 import androidx.core.view.WindowInsetsCompat
 import app.omnivore.omnivore.ui.auth.LoginViewModel
 import app.omnivore.omnivore.ui.library.LibraryViewModel
-import app.omnivore.omnivore.ui.reader.WebReaderViewModel
 import app.omnivore.omnivore.ui.root.RootView
 import app.omnivore.omnivore.ui.theme.OmnivoreTheme
 import com.pspdfkit.PSPDFKit
@@ -32,7 +31,6 @@ class MainActivity : ComponentActivity() {
 
     val loginViewModel: LoginViewModel by viewModels()
     val libraryViewModel: LibraryViewModel by viewModels()
-    val webReaderViewModel: WebReaderViewModel by viewModels()
 
     val context = this
 
@@ -53,7 +51,7 @@ class MainActivity : ComponentActivity() {
             .fillMaxSize()
             .background(color = Color.Black)
         ) {
-          RootView(loginViewModel, libraryViewModel, webReaderViewModel)
+          RootView(loginViewModel, libraryViewModel)
         }
       }
     }

android/Omnivore/app/src/main/java/app/omnivore/omnivore/dataService/DataService.kt

@@ -4,6 +4,7 @@ import android.content.Context
 import androidx.room.Room
 import app.omnivore.omnivore.networking.*
 import app.omnivore.omnivore.persistence.AppDatabase
+import kotlinx.coroutines.*
 import javax.inject.Inject
 
 class DataService @Inject constructor(
@@ -14,4 +15,10 @@ class DataService @Inject constructor(
     context,
     AppDatabase::class.java, "omnivore-database"
   ).build()
+
+  fun clearDatabase() {
+    CoroutineScope(Dispatchers.IO).launch {
+      db.clearAllTables()
+    }
+  }
 }

android/Omnivore/app/src/main/java/app/omnivore/omnivore/dataService/LibraryLiveData.kt

@@ -0,0 +1,120 @@
+package app.omnivore.omnivore.dataService
+
+import androidx.lifecycle.LiveData
+import androidx.lifecycle.MediatorLiveData
+import app.omnivore.omnivore.persistence.entities.SavedItemCardDataWithLabels
+import app.omnivore.omnivore.persistence.entities.SavedItemLabel
+import app.omnivore.omnivore.ui.library.SavedItemFilter
+import app.omnivore.omnivore.ui.library.SavedItemSortFilter
+
+fun DataService.libraryLiveData(
+  primaryFilter: SavedItemFilter,
+  sortFilter: SavedItemSortFilter,
+  labels: List<SavedItemLabel>
+): LiveData<List<SavedItemCardDataWithLabels>> {
+  val mediatorLiveData = MediatorLiveData<List<SavedItemCardDataWithLabels>>()
+  val queryParams = LibraryLiveDataQueryParams.make(primaryFilter)
+
+  val libraryLiveData = when (sortFilter) {
+    SavedItemSortFilter.NEWEST -> db.savedItemDao().getLibraryLiveData(
+      archiveFilter = queryParams.archiveFilter
+    )
+    SavedItemSortFilter.OLDEST -> db.savedItemDao().getLibraryLiveDataSortedByOldest(
+      archiveFilter = queryParams.archiveFilter
+    )
+    SavedItemSortFilter.RECENTLY_READ -> db.savedItemDao().getLibraryLiveDataSortedByRecentlyRead(
+      archiveFilter = queryParams.archiveFilter
+    )
+    SavedItemSortFilter.RECENTLY_PUBLISHED -> db.savedItemDao().getLibraryLiveDataSortedByRecentlyPublished(
+      archiveFilter = queryParams.archiveFilter
+    )
+  }
+
+  mediatorLiveData.addSource(libraryLiveData) { result ->
+    when (primaryFilter) {
+      SavedItemFilter.INBOX -> {
+        mediatorLiveData.value = result
+      }
+      SavedItemFilter.READ_LATER -> {
+        mediatorLiveData.value = result.filter { item ->
+          !item.labels.any { it.name.lowercase() == "newsletter" }
+        }
+      }
+      SavedItemFilter.NEWSLETTERS -> {
+        mediatorLiveData.value = result.filter { item ->
+          item.labels.any { it.name.lowercase() == "newsletter" }
+        }
+      }
+      SavedItemFilter.RECOMMENDED -> {
+        mediatorLiveData.value = result // TODO: "recommendations.@count > 0"
+      }
+      SavedItemFilter.ALL -> {
+        mediatorLiveData.value = result
+      }
+      SavedItemFilter.ARCHIVED -> {
+        mediatorLiveData.value = result
+      }
+      SavedItemFilter.HAS_HIGHLIGHTS -> {
+        mediatorLiveData.value = result // TODO: "highlights.@count > 0"
+      }
+      SavedItemFilter.FILES -> {
+        mediatorLiveData.value = result.filter { item ->
+          item.cardData.contentReader == "PDF"
+        }
+      }
+    }
+  }
+
+  return mediatorLiveData
+}
+
+private data class LibraryLiveDataQueryParams(
+  val archiveFilter: Int
+) {
+  companion object {
+    fun make(savedItemFilter: SavedItemFilter): LibraryLiveDataQueryParams {
+      return when (savedItemFilter) {
+        SavedItemFilter.INBOX -> {
+          LibraryLiveDataQueryParams(
+            archiveFilter = 1, // Filter out items marked as archive
+          )
+        }
+        SavedItemFilter.READ_LATER -> {
+          LibraryLiveDataQueryParams(
+            archiveFilter = 1, // Filter out items marked as archive
+          )
+        }
+        SavedItemFilter.NEWSLETTERS -> {
+          LibraryLiveDataQueryParams(
+            archiveFilter = 1, // Filter out items marked as archive
+          )
+        }
+        SavedItemFilter.RECOMMENDED -> {
+          LibraryLiveDataQueryParams(
+            archiveFilter = 1, // Filter out items marked as archive
+          )
+        }
+        SavedItemFilter.ALL -> {
+          LibraryLiveDataQueryParams(
+            archiveFilter = 2, // Don't filter anything out (2 will not match anything)
+          )
+        }
+        SavedItemFilter.ARCHIVED -> {
+          LibraryLiveDataQueryParams(
+            archiveFilter = 0, // Filter out items not marked as archived
+          )
+        }
+        SavedItemFilter.HAS_HIGHLIGHTS -> {
+          LibraryLiveDataQueryParams(
+            archiveFilter = 1, // Filter out items marked as archive
+          )
+        }
+        SavedItemFilter.FILES -> {
+          LibraryLiveDataQueryParams(
+            archiveFilter = 1, // Filter out items marked as archive
+          )
+        }
+      }
+    }
+  }
+}

android/Omnivore/app/src/main/java/app/omnivore/omnivore/persistence/entities/SavedItem.kt

@@ -3,6 +3,9 @@ package app.omnivore.omnivore.persistence.entities
 import androidx.core.net.toUri
 import androidx.lifecycle.LiveData
 import androidx.room.*
+import app.omnivore.omnivore.BuildConfig
+import app.omnivore.omnivore.models.ServerSyncStatus
+import app.omnivore.omnivore.ui.library.SavedItemSortFilter
 import java.util.*
 
 @Entity
@@ -86,9 +89,6 @@ data class SavedItemCardData(
 
 @Dao
 interface SavedItemDao {
-  @Query("SELECT savedItemId, slug, publisherURLString, title, author, imageURLString, isArchived, pageURLString, contentReader FROM SavedItem ORDER BY savedAt DESC")
-  fun getLibraryLiveData(): LiveData<List<SavedItemCardData>>
-
   @Query("SELECT * FROM savedItem")
   fun getAll(): List<SavedItem>
 
@@ -114,6 +114,43 @@ interface SavedItemDao {
   fun update(savedItem: SavedItem)
 
   @Transaction
-  @Query("SELECT savedItemId, slug, publisherURLString, title, author, imageURLString, isArchived, pageURLString, contentReader FROM SavedItem ORDER BY savedAt DESC")
-  fun getLibraryLiveDataWithLabels(): LiveData<List<SavedItemCardDataWithLabels>>
+  @Query(
+    "SELECT ${SavedItemQueryConstants.columns} " +
+      "FROM SavedItem " +
+      "WHERE serverSyncStatus != 2 AND isArchived != :archiveFilter " +
+      "ORDER BY savedAt DESC"
+  )
+  fun getLibraryLiveData(archiveFilter: Int): LiveData<List<SavedItemCardDataWithLabels>>
+
+  @Transaction
+  @Query(
+    "SELECT ${SavedItemQueryConstants.columns} " +
+      "FROM SavedItem " +
+      "WHERE serverSyncStatus != 2 AND isArchived != :archiveFilter " +
+      "ORDER BY savedAt ASC"
+  )
+  fun getLibraryLiveDataSortedByOldest(archiveFilter: Int): LiveData<List<SavedItemCardDataWithLabels>>
+
+  @Transaction
+  @Query(
+    "SELECT ${SavedItemQueryConstants.columns} " +
+      "FROM SavedItem " +
+      "WHERE serverSyncStatus != 2 AND isArchived != :archiveFilter " +
+      "ORDER BY readAt DESC, savedAt DESC"
+  )
+  fun getLibraryLiveDataSortedByRecentlyRead(archiveFilter: Int): LiveData<List<SavedItemCardDataWithLabels>>
+
+  @Transaction
+  @Query(
+    "SELECT ${SavedItemQueryConstants.columns} " +
+      "FROM SavedItem " +
+      "WHERE serverSyncStatus != 2 AND isArchived != :archiveFilter " +
+      "ORDER BY publishDate DESC"
+  )
+  fun getLibraryLiveDataSortedByRecentlyPublished(archiveFilter: Int): LiveData<List<SavedItemCardDataWithLabels>>
+}
+
+
+object SavedItemQueryConstants {
+  const val columns = "savedItemId, slug, publisherURLString, title, author, imageURLString, isArchived, pageURLString, contentReader "
 }

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/auth/LoginViewModel.kt

@@ -5,6 +5,7 @@ import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.setValue
 import androidx.lifecycle.*
 import app.omnivore.omnivore.*
+import app.omnivore.omnivore.dataService.DataService
 import app.omnivore.omnivore.graphql.generated.ValidateUsernameQuery
 import app.omnivore.omnivore.networking.Networker
 import app.omnivore.omnivore.networking.viewer
@@ -14,11 +15,8 @@ import com.google.android.gms.common.api.ApiException
 import com.google.android.gms.tasks.Task
 import dagger.hilt.android.lifecycle.HiltViewModel
 import io.intercom.android.sdk.Intercom
-import kotlinx.coroutines.Job
-import kotlinx.coroutines.delay
+import kotlinx.coroutines.*
 import kotlinx.coroutines.flow.distinctUntilChanged
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.runBlocking
 import java.util.regex.Pattern
 import javax.inject.Inject
 
@@ -38,7 +36,8 @@ data class PendingEmailUserCreds(
 class LoginViewModel @Inject constructor(
   private val datastoreRepo: DatastoreRepository,
   private val eventTracker: EventTracker,
-  private val networker: Networker
+  private val networker: Networker,
+  private val dataService: DataService
 ): ViewModel() {
   private var validateUsernameJob: Job? = null
 
@@ -218,7 +217,7 @@ class LoginViewModel @Inject constructor(
       isLoading = false
 
       if (result.errorBody() != null) {
-        errorMessage = "Something went wrong. Please check your  and try again"
+        errorMessage = "Something went wrong. Please check your entries and try again"
       } else {
         pendingEmailUserCreds = PendingEmailUserCreds(email, password)
       }
@@ -271,6 +270,7 @@ class LoginViewModel @Inject constructor(
   fun logout() {
     viewModelScope.launch {
       datastoreRepo.clear()
+      dataService.clearDatabase()
       Intercom.client().logout()
     }
   }

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryFilterBar.kt

@@ -0,0 +1,65 @@
+package app.omnivore.omnivore.ui.library
+
+import androidx.compose.foundation.layout.*
+import androidx.compose.material.icons.Icons
+import androidx.compose.material.icons.filled.ArrowDropDown
+import androidx.compose.material3.*
+import androidx.compose.runtime.*
+import androidx.compose.runtime.livedata.observeAsState
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.unit.dp
+
+@OptIn(ExperimentalMaterial3Api::class)
+@Composable
+fun LibraryFilterBar(viewModel: LibraryViewModel) {
+  var isSavedItemFilterMenuExpanded by remember { mutableStateOf(false) }
+  val activeSavedItemFilter: SavedItemFilter by viewModel.appliedFilterLiveData.observeAsState(SavedItemFilter.INBOX)
+
+  var isSavedItemSortFilterMenuExpanded by remember { mutableStateOf(false) }
+  val activeSavedItemSortFilter: SavedItemSortFilter by viewModel.appliedSortFilterLiveData.observeAsState(SavedItemSortFilter.NEWEST)
+
+  Column {
+    Row(
+      horizontalArrangement = Arrangement.Start,
+      verticalAlignment = Alignment.CenterVertically,
+      modifier = Modifier
+        .fillMaxWidth()
+    ) {
+      AssistChip(
+        onClick = { isSavedItemFilterMenuExpanded = true },
+        label = { Text(activeSavedItemFilter.displayText) },
+        trailingIcon = {
+          Icon(
+            Icons.Default.ArrowDropDown,
+            contentDescription = "drop down button to change primary library filter"
+          )
+        },
+        modifier = Modifier.padding(end = 6.dp)
+      )
+      AssistChip(
+        onClick = { isSavedItemSortFilterMenuExpanded = true },
+        label = { Text(activeSavedItemSortFilter.displayText) },
+        trailingIcon = {
+          Icon(
+            Icons.Default.ArrowDropDown,
+            contentDescription = "drop down button to change library sort order"
+          )
+        },
+        modifier = Modifier.padding(end = 6.dp)
+      )
+    }
+
+    SavedItemFilterContextMenu(
+      isExpanded = isSavedItemFilterMenuExpanded,
+      onDismiss = { isSavedItemFilterMenuExpanded = false },
+      actionHandler = { viewModel.updateSavedItemFilter(it) }
+    )
+
+    SavedItemSortFilterContextMenu(
+      isExpanded = isSavedItemSortFilterMenuExpanded,
+      onDismiss = { isSavedItemSortFilterMenuExpanded = false },
+      actionHandler = { viewModel.updateSavedItemSortFilter(it) }
+    )
+  }
+}

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryView.kt

@@ -18,14 +18,12 @@ import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.platform.LocalContext
 import androidx.compose.ui.unit.dp
-import androidx.compose.ui.unit.sp
 import androidx.navigation.NavHostController
 import app.omnivore.omnivore.Routes
-import app.omnivore.omnivore.persistence.entities.SavedItemAndSavedItemLabelCrossRef
-import app.omnivore.omnivore.persistence.entities.SavedItemCardData
 import app.omnivore.omnivore.persistence.entities.SavedItemCardDataWithLabels
 import app.omnivore.omnivore.ui.savedItemViews.SavedItemCard
 import app.omnivore.omnivore.ui.reader.PDFReaderActivity
+import app.omnivore.omnivore.ui.reader.WebReaderLoadingContainerActivity
 import kotlinx.coroutines.flow.distinctUntilChanged
 
 
@@ -35,20 +33,16 @@ fun LibraryView(
   libraryViewModel: LibraryViewModel,
   navController: NavHostController
 ) {
-  val searchText: String by libraryViewModel.searchTextLiveData.observeAsState("")
-
   Scaffold(
     topBar = {
       SearchBar(
-        searchText = searchText,
-        onSearchTextChanged = { libraryViewModel.updateSearchText(it) },
+        libraryViewModel = libraryViewModel,
         onSettingsIconClick = { navController.navigate(Routes.Settings.route) }
       )
     }
   ) { paddingValues ->
     LibraryViewContent(
       libraryViewModel,
-      navController,
       modifier = Modifier
         .padding(
           top = paddingValues.calculateTopPadding(),
@@ -60,11 +54,7 @@ fun LibraryView(
 
 @OptIn(ExperimentalMaterialApi::class)
 @Composable
-fun LibraryViewContent(
-  libraryViewModel: LibraryViewModel,
-  navController: NavHostController,
-  modifier: Modifier
-) {
+fun LibraryViewContent(libraryViewModel: LibraryViewModel, modifier: Modifier) {
   val context = LocalContext.current
   val listState = rememberLazyListState()
 
@@ -91,17 +81,19 @@ fun LibraryViewContent(
         .fillMaxSize()
         .padding(horizontal = 6.dp)
     ) {
-      items(if (searchText.isNotEmpty()) searchedCardsData else cardsData) { cardDataWithLabels ->
+      if (!libraryViewModel.showSearchField) {
+        item {
+          LibraryFilterBar(libraryViewModel)
+        }
+      }
+      items(if (libraryViewModel.showSearchField) searchedCardsData else cardsData) { cardDataWithLabels ->
         SavedItemCard(
           cardData = cardDataWithLabels.cardData,
           onClickHandler = {
-            if (cardDataWithLabels.cardData.isPDF()) {
-              val intent = Intent(context, PDFReaderActivity::class.java)
-              intent.putExtra("SAVED_ITEM_SLUG", cardDataWithLabels.cardData.slug)
-              context.startActivity(intent)
-            } else {
-              navController.navigate("WebReader/${cardDataWithLabels.cardData.slug}")
-            }
+            val activityClass = if (cardDataWithLabels.cardData.isPDF()) PDFReaderActivity::class.java else WebReaderLoadingContainerActivity::class.java
+            val intent = Intent(context, activityClass)
+            intent.putExtra("SAVED_ITEM_SLUG", cardDataWithLabels.cardData.slug)
+            context.startActivity(intent)
           },
           actionHandler = { libraryViewModel.handleSavedItemAction(cardDataWithLabels.cardData.savedItemId, it) }
         )

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/LibraryViewModel.kt

@@ -3,18 +3,22 @@ package app.omnivore.omnivore.ui.library
 import android.util.Log
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
 import androidx.compose.runtime.setValue
+import androidx.lifecycle.LiveData
+import androidx.lifecycle.MediatorLiveData
 import androidx.lifecycle.MutableLiveData
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import app.omnivore.omnivore.*
 import app.omnivore.omnivore.dataService.*
 import app.omnivore.omnivore.networking.*
-import app.omnivore.omnivore.persistence.entities.SavedItemCardData
+import app.omnivore.omnivore.persistence.entities.SavedItem
 import app.omnivore.omnivore.persistence.entities.SavedItemCardDataWithLabels
+import app.omnivore.omnivore.ui.reader.WebFont
 import dagger.hilt.android.lifecycle.HiltViewModel
 import kotlinx.coroutines.*
-import java.time.LocalDateTime
+import java.time.Instant
 import javax.inject.Inject
 
 @HiltViewModel
@@ -33,9 +37,43 @@ class LibraryViewModel @Inject constructor(
   // Live Data
   val searchTextLiveData = MutableLiveData("")
   val searchItemsLiveData = MutableLiveData<List<SavedItemCardDataWithLabels>>(listOf())
-  val itemsLiveData = dataService.db.savedItemDao().getLibraryLiveDataWithLabels()
+  private var itemsLiveDataInternal = dataService.libraryLiveData(SavedItemFilter.INBOX, SavedItemSortFilter.NEWEST, listOf())
+  val itemsLiveData = MediatorLiveData<List<SavedItemCardDataWithLabels>>()
+  val appliedFilterLiveData = MutableLiveData(SavedItemFilter.INBOX)
+  val appliedSortFilterLiveData = MutableLiveData(SavedItemSortFilter.NEWEST)
 
   var isRefreshing by mutableStateOf(false)
+  var showSearchField by mutableStateOf(false)
+  var hasLoadedInitialFilters = false
+
+  fun loadInitialFilterValues() {
+    if (hasLoadedInitialFilters) { return }
+    hasLoadedInitialFilters = false
+
+    runBlocking {
+      datastoreRepo.getString(DatastoreKeys.lastUsedSavedItemFilter)?.let { str ->
+        try {
+          val filter = SavedItemFilter.values().first { it.rawValue == str }
+          appliedFilterLiveData.postValue(filter)
+        } catch (e: Exception) {
+          Log.d("error", "invalid filter value stored in datastore repo: $e")
+        }
+      }
+
+      datastoreRepo.getString(DatastoreKeys.lastUsedSavedItemSortFilter)?.let { str ->
+        try {
+          val filter = SavedItemSortFilter.values().first { it.rawValue == str }
+          appliedSortFilterLiveData.postValue(filter)
+        } catch (e: Exception) {
+          Log.d("error", "invalid sort filter value stored in datastore repo: $e")
+        }
+      }
+    }
+
+    viewModelScope.launch {
+      handleFilterChanges()
+    }
+  }
 
   fun updateSearchText(text: String) {
     searchTextLiveData.value = text
@@ -52,13 +90,19 @@ class LibraryViewModel @Inject constructor(
     load(true)
   }
 
-  fun getLastSyncTime(): LocalDateTime? = runBlocking {
+  fun getLastSyncTime(): Instant? = runBlocking {
     datastoreRepo.getString(DatastoreKeys.libraryLastSyncTimestamp)?.let {
-      LocalDateTime.parse(it)
+      try {
+        return@let Instant.parse(it)
+      } catch (e: Exception) {
+        return@let null
+      }
     }
   }
 
   fun load(clearPreviousSearch: Boolean = false) {
+    loadInitialFilterValues()
+
     viewModelScope.launch {
       if (searchTextLiveData.value != "") {
         performSearch(clearPreviousSearch)
@@ -68,16 +112,41 @@ class LibraryViewModel @Inject constructor(
     }
   }
 
-  private suspend fun syncItems() {
-    val syncStart = LocalDateTime.now()
-    val lastSyncDate = getLastSyncTime() ?: LocalDateTime.MIN
-
-    CoroutineScope(Dispatchers.Main).launch {
-      isRefreshing = false
+  fun updateSavedItemFilter(filter: SavedItemFilter) {
+    viewModelScope.launch {
+      datastoreRepo.putString(DatastoreKeys.lastUsedSavedItemFilter, filter.rawValue)
+      appliedFilterLiveData.value = filter
+      handleFilterChanges()
     }
+  }
+
+  fun updateSavedItemSortFilter(filter: SavedItemSortFilter) {
+    viewModelScope.launch {
+      datastoreRepo.putString(DatastoreKeys.lastUsedSavedItemSortFilter, filter.rawValue)
+      appliedSortFilterLiveData.value = filter
+      handleFilterChanges()
+    }
+  }
+
+  suspend fun handleFilterChanges() {
+    if (searchTextLiveData.value != "") {
+      performSearch(true)
+    } else if (appliedSortFilterLiveData.value != null && appliedFilterLiveData.value != null) {
+      itemsLiveDataInternal = dataService.libraryLiveData(appliedFilterLiveData.value!!, appliedSortFilterLiveData.value!!, listOf())
+      itemsLiveData.removeSource(itemsLiveDataInternal)
+      itemsLiveData.addSource(itemsLiveDataInternal, itemsLiveData::setValue)
+    }
+  }
+
+  private suspend fun syncItems() {
+    val syncStart = Instant.now()
+    val lastSyncDate = getLastSyncTime() ?: Instant.MIN
 
     withContext(Dispatchers.IO) {
       performItemSync(cursor = null, since = lastSyncDate.toString(), count = 0, startTime = syncStart.toString())
+      CoroutineScope(Dispatchers.Main).launch {
+        isRefreshing = false
+      }
     }
   }
 
@@ -118,7 +187,7 @@ class LibraryViewModel @Inject constructor(
     searchIdx += 1
 
     // Execute the search
-    val searchResult = networker.typeaheadSearch(searchTextLiveData.value ?: "")
+    val searchResult = networker.typeaheadSearch(searchQueryString())
 
     // Search results aren't guaranteed to return in order so this
     // will discard old results that are returned while a user is typing.
@@ -159,6 +228,19 @@ class LibraryViewModel @Inject constructor(
       }
     }
   }
+
+  private fun searchQueryString(): String {
+    return searchTextLiveData.value ?: ""
+    // Unused code for typeahead search
+//    var query = "${appliedFilterLiveData.value?.queryString} ${appliedSortFilterLiveData.value?.queryString}"
+//    val searchText = searchTextLiveData.value ?: ""
+//
+//    if (searchText.isNotEmpty()) {
+//      query += " $searchText"
+//    }
+//
+//    return query
+  }
 }
 
 enum class SavedItemAction {

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/SavedItemFilter.kt

@@ -0,0 +1,48 @@
+package app.omnivore.omnivore.ui.library
+
+import androidx.compose.material3.DropdownMenu
+import androidx.compose.material3.DropdownMenuItem
+import androidx.compose.material3.Text
+import androidx.compose.runtime.Composable
+
+enum class SavedItemFilter(val displayText: String, val rawValue: String, val queryString: String) {
+  INBOX("Inbox", rawValue = "inbox", "in:inbox"),
+  READ_LATER("Read Later", "readlater", "in:inbox -label:Newsletter"),
+  NEWSLETTERS("Newsletters", "newsletters", "in:inbox label:Newsletter"),
+  RECOMMENDED("Recommended", "recommended", "recommendedBy:*"),
+  ALL("All", "all", "in:all"),
+  ARCHIVED("Archived", "archived", "in:archive"),
+  HAS_HIGHLIGHTS("Highlighted", "hasHighlights", "has:highlights"),
+  FILES("Files", "files", "type:file"),
+}
+
+@Composable
+fun SavedItemFilterContextMenu(
+  isExpanded: Boolean,
+  onDismiss: () -> Unit,
+  actionHandler: (SavedItemFilter) -> Unit
+) {
+  DropdownMenu(
+    expanded = isExpanded,
+    onDismissRequest = onDismiss
+  ) {
+    // Displaying only a subset of filters until we figure out the Room DB queries (and labels)
+//    SavedItemFilter.values().forEach {
+      listOf(
+        SavedItemFilter.INBOX,
+        SavedItemFilter.READ_LATER,
+        SavedItemFilter.NEWSLETTERS,
+        SavedItemFilter.ALL,
+        SavedItemFilter.ARCHIVED,
+        SavedItemFilter.FILES
+      ).forEach {
+      DropdownMenuItem(
+        text = { Text(it.displayText) },
+        onClick = {
+          actionHandler(it)
+          onDismiss()
+        }
+      )
+    }
+  }
+}

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/SavedItemSortFilter.kt

@@ -0,0 +1,35 @@
+package app.omnivore.omnivore.ui.library
+
+import androidx.compose.material3.DropdownMenu
+import androidx.compose.material3.DropdownMenuItem
+import androidx.compose.material3.Text
+import androidx.compose.runtime.Composable
+
+enum class SavedItemSortFilter(val displayText: String, val rawValue: String, val queryString: String) {
+  NEWEST("Newest", rawValue = "newest", "sort:saved"),
+  OLDEST("Oldest", rawValue = "oldest", "sort:saved-ASC"),
+  RECENTLY_READ("Recently Read", rawValue = "recentlyRead", "sort:read"),
+  RECENTLY_PUBLISHED("Recently Published", rawValue = "recentlyPublished", "sort:published"),
+}
+
+@Composable
+fun SavedItemSortFilterContextMenu(
+  isExpanded: Boolean,
+  onDismiss: () -> Unit,
+  actionHandler: (SavedItemSortFilter) -> Unit
+) {
+  DropdownMenu(
+    expanded = isExpanded,
+    onDismissRequest = onDismiss
+  ) {
+    SavedItemSortFilter.values().forEach {
+      DropdownMenuItem(
+        text = { Text(it.displayText) },
+        onClick = {
+          actionHandler(it)
+          onDismiss()
+        }
+      )
+    }
+  }
+}

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/library/SearchBar.kt

@@ -15,6 +15,7 @@ import androidx.compose.material.icons.filled.Search
 import androidx.compose.material.icons.filled.Settings
 import androidx.compose.material3.*
 import androidx.compose.runtime.*
+import androidx.compose.runtime.livedata.observeAsState
 import androidx.compose.ui.ExperimentalComposeUiApi
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.focus.FocusRequester
@@ -27,16 +28,15 @@ import androidx.compose.ui.unit.dp
 @OptIn(ExperimentalMaterial3Api::class)
 @Composable
 fun SearchBar(
-  searchText: String,
-  onSearchTextChanged: (String) -> Unit,
+  libraryViewModel: LibraryViewModel,
   onSettingsIconClick: () -> Unit
 ) {
-  var showSearchField by remember { mutableStateOf(searchText != "") }
+  val searchText: String by libraryViewModel.searchTextLiveData.observeAsState("")
 
   SmallTopAppBar(
     title = {
-      if (showSearchField) {
-        SearchField(searchText, onSearchTextChanged)
+      if (libraryViewModel.showSearchField) {
+        SearchField(searchText) { libraryViewModel.updateSearchText(it) }
       } else {
         Text("Library")
       }
@@ -45,18 +45,18 @@ fun SearchBar(
       containerColor = MaterialTheme.colorScheme.surfaceVariant
     ),
     actions = {
-      if (showSearchField) {
+      if (libraryViewModel.showSearchField) {
         Text(
           text = "Cancel",
           modifier = Modifier
             .clickable {
-              onSearchTextChanged("")
-              showSearchField = false
+              libraryViewModel.updateSearchText("")
+              libraryViewModel.showSearchField = false
             }
             .padding(horizontal = 6.dp)
         )
       } else {
-        IconButton(onClick = { showSearchField = true }) {
+        IconButton(onClick = { libraryViewModel.showSearchField = true }) {
           Icon(
             imageVector = Icons.Filled.Search,
             contentDescription = null

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReader.kt

@@ -4,42 +4,28 @@ import android.annotation.SuppressLint
 import android.content.ClipData
 import android.content.ClipboardManager
 import android.content.Context
+import android.graphics.Bitmap
 import android.graphics.Rect
 import android.util.Log
 import android.view.*
+import android.view.View.OnScrollChangeListener
+import android.view.ViewTreeObserver.OnScrollChangedListener
 import android.webkit.JavascriptInterface
 import android.webkit.WebView
 import android.webkit.WebViewClient
-import androidx.activity.compose.LocalOnBackPressedDispatcherOwner
 import androidx.compose.foundation.isSystemInDarkTheme
 import androidx.compose.foundation.layout.*
-import androidx.compose.foundation.verticalScroll
-import androidx.compose.material.ModalBottomSheetValue
-import androidx.compose.material.TopAppBar
-import androidx.compose.material.icons.Icons
-import androidx.compose.material.icons.filled.Menu
-import androidx.compose.material.icons.filled.Settings
 import androidx.compose.material3.*
 import androidx.compose.runtime.*
 import androidx.compose.runtime.livedata.observeAsState
-import androidx.compose.ui.Modifier
-import androidx.compose.ui.geometry.Offset
 import androidx.compose.ui.graphics.Color
-import androidx.compose.ui.input.nestedscroll.NestedScrollConnection
-import androidx.compose.ui.input.nestedscroll.NestedScrollSource
-import androidx.compose.ui.input.nestedscroll.nestedScroll
-import androidx.compose.ui.platform.LocalDensity
-import androidx.compose.ui.unit.dp
 import androidx.compose.ui.viewinterop.AndroidView
 import app.omnivore.omnivore.R
-import app.omnivore.omnivore.ui.save.SaveSheetActivityBase
-import app.omnivore.omnivore.ui.savedItemViews.SavedItemContextMenu
 import com.google.gson.Gson
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 import java.util.*
-import kotlin.math.roundToInt
 
 @SuppressLint("SetJavaScriptEnabled")
 @Composable
@@ -81,7 +67,14 @@ fun WebReader(
         settings.allowFileAccess = true
         settings.domStorageEnabled = true
 
+        alpha = 0.0f
+
         webViewClient = object : WebViewClient() {
+          override fun onPageFinished(view: WebView?, url: String?) {
+            super.onPageFinished(view, url)
+            viewModel?.showNavBar()
+            view?.animate()?.alpha(1.0f)?.duration = 200
+          }
         }
 
         val javascriptInterface = AndroidWebKitMessenger { actionID, json ->
@@ -99,7 +92,6 @@ fun WebReader(
             }
             "existingHighlightTap" -> {
               val tapCoordinates = Gson().fromJson(json, TapCoordinates::class.java)
-              Log.d("wv", "receive existing highlight tap action: $tapCoordinates")
               CoroutineScope(Dispatchers.Main).launch {
                 webReaderViewModel.hasTappedExistingHighlight = true
                 webReaderViewModel.lastTapCoordinates = tapCoordinates
@@ -142,9 +134,14 @@ fun WebReader(
   }
 }
 
-class OmnivoreWebView(context: Context) : WebView(context) {
+class OmnivoreWebView(context: Context) : WebView(context), OnScrollChangeListener {
   var viewModel: WebReaderViewModel? = null
   var actionMode: ActionMode? = null
+  val density = resources.displayMetrics.density
+
+  init {
+    setOnScrollChangeListener(this)
+  }
 
   private val actionModeCallback = object : ActionMode.Callback2() {
     // Called when the action mode is created; startActionMode() was called
@@ -229,12 +226,10 @@ class OmnivoreWebView(context: Context) : WebView(context) {
     override fun onGetContentRect(mode: ActionMode?, view: View?, outRect: Rect?) {
       Log.d("wv", "outRect: $outRect, View: $view")
       if (viewModel?.lastTapCoordinates != null) {
-        val scrollYOffset = viewModel?.scrollState?.value ?: 0
-        val xValue = viewModel!!.lastTapCoordinates!!.tapX.toInt()
-        val yValue = viewModel!!.lastTapCoordinates!!.tapY.toInt() + scrollYOffset + (viewModel?.currentToolbarHeight ?: 0)
+        val xValue = (viewModel!!.lastTapCoordinates!!.tapX * density).toInt()
+        val yValue = (viewModel!!.lastTapCoordinates!!.tapY * density).toInt()
         val rect = Rect(xValue, yValue, xValue, yValue)
 
-        Log.d("wvt", "scrollState: ${viewModel?.scrollState?.value}, bar height: ${viewModel?.currentToolbarHeight}")
         Log.d("wvt", "setting rect based on last tapped rect: ${viewModel?.lastTapCoordinates.toString()}")
         Log.d("wvt", "rect: $rect")
 
@@ -262,6 +257,10 @@ class OmnivoreWebView(context: Context) : WebView(context) {
     Log.d("wv", "startActionMode:type called")
     return super.startActionMode(actionModeCallback, type)
   }
+
+  override fun onScrollChange(view: View?, x: Int, y: Int, oldX: Int, oldY: Int) {
+    viewModel?.onScrollChange((oldY - y).toFloat())
+  }
 }
 
 class AndroidWebKitMessenger(val messageHandler: (String, String) -> Unit) {

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderLoadingContainer.kt

@@ -2,20 +2,18 @@ package app.omnivore.omnivore.ui.reader
 
 import android.content.Intent
 import android.os.Bundle
+import android.util.Log
 import android.view.View
 import androidx.activity.ComponentActivity
 import androidx.activity.compose.LocalOnBackPressedDispatcherOwner
 import androidx.activity.compose.setContent
 import androidx.activity.viewModels
 import androidx.compose.foundation.background
-import androidx.compose.foundation.clickable
 import androidx.compose.foundation.isSystemInDarkTheme
 import androidx.compose.foundation.layout.*
-import androidx.compose.foundation.verticalScroll
 import androidx.compose.material.TopAppBar
 import androidx.compose.material.icons.Icons
 import androidx.compose.material.icons.filled.Home
-import androidx.compose.material.icons.filled.List
 import androidx.compose.material.icons.filled.Menu
 import androidx.compose.material.icons.filled.Settings
 import androidx.compose.material3.Icon
@@ -26,11 +24,7 @@ import androidx.compose.runtime.*
 import androidx.compose.runtime.livedata.observeAsState
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
-import androidx.compose.ui.geometry.Offset
 import androidx.compose.ui.graphics.Color
-import androidx.compose.ui.input.nestedscroll.NestedScrollConnection
-import androidx.compose.ui.input.nestedscroll.NestedScrollSource
-import androidx.compose.ui.input.nestedscroll.nestedScroll
 import androidx.compose.ui.platform.LocalDensity
 import androidx.compose.ui.unit.dp
 import androidx.core.view.ViewCompat
@@ -50,31 +44,13 @@ class WebReaderLoadingContainerActivity: ComponentActivity() {
 
   override fun onCreate(savedInstanceState: Bundle?) {
     super.onCreate(savedInstanceState)
-    val requestID = intent.getStringExtra("SAVED_ITEM_REQUEST_ID") ?: ""
+    val requestID = intent.getStringExtra("SAVED_ITEM_REQUEST_ID")
+    val slug = intent.getStringExtra("SAVED_ITEM_SLUG")
 
     setContent {
       val systemUiController = rememberSystemUiController()
       val useDarkIcons = !isSystemInDarkTheme()
 
-      OmnivoreTheme {
-        Box(
-          modifier = Modifier
-            .fillMaxSize()
-            .background(color = Color.Black)
-            .systemBarsPadding()
-        ) {
-          if (viewModel.hasFetchError.value == true) {
-            Text("We were unable to fetch your content.")
-          } else {
-            WebReaderLoadingContainer(
-              requestID = requestID,
-              onLibraryIconTap = { startMainActivity() },
-              webReaderViewModel = viewModel
-            )
-          }
-        }
-      }
-
       DisposableEffect(systemUiController, useDarkIcons) {
         systemUiController.setSystemBarsColor(
           color = Color.Black,
@@ -83,6 +59,25 @@ class WebReaderLoadingContainerActivity: ComponentActivity() {
 
         onDispose {}
       }
+
+      OmnivoreTheme {
+        Box(
+          modifier = Modifier
+            .fillMaxSize()
+            .background(color = Color.Black)
+        ) {
+          if (viewModel.hasFetchError.value == true) {
+            Text("We were unable to fetch your content.")
+          } else {
+            WebReaderLoadingContainer(
+              requestID = requestID,
+              slug = slug,
+              onLibraryIconTap = if (requestID != null) { { startMainActivity() } } else null,
+              webReaderViewModel = viewModel
+            )
+          }
+        }
+      }
     }
 
     // animate the view up when keyboard appears
@@ -111,54 +106,31 @@ fun WebReaderLoadingContainer(slug: String? = null, requestID: String? = null, o
   val webReaderParams: WebReaderParams? by webReaderViewModel.webReaderParamsLiveData.observeAsState(null)
   val annotation: String? by webReaderViewModel.annotationLiveData.observeAsState(null)
   val shouldPopView: Boolean by webReaderViewModel.shouldPopViewLiveData.observeAsState(false)
+  val toolbarHeightPx: Float by webReaderViewModel.currentToolbarHeightLiveData.observeAsState(0.0f)
 
   val maxToolbarHeight = 48.dp
-  val maxToolbarHeightPx = with(LocalDensity.current) { maxToolbarHeight.roundToPx().toFloat() }
-  val toolbarHeightPx = remember { mutableStateOf(maxToolbarHeightPx) }
+  val backgroundColor = if (isSystemInDarkTheme()) Color.Black else Color.White
+  webReaderViewModel.maxToolbarHeightPx = with(LocalDensity.current) { maxToolbarHeight.roundToPx().toFloat() }
+  webReaderViewModel.loadItem(slug = slug, requestID = requestID)
 
-  // Create a connection to the nested scroll system and listen to the scroll happening inside child Column
-  val nestedScrollConnection = remember {
-    object : NestedScrollConnection {
-      override fun onPreScroll(available: Offset, source: NestedScrollSource): Offset {
-        val delta = available.y
-        val newHeight = toolbarHeightPx.value + delta
-        toolbarHeightPx.value = newHeight.coerceIn(0f, maxToolbarHeightPx)
-        return Offset.Zero
-      }
-    }
-  }
-
-  if (webReaderParams == null) {
-    webReaderViewModel.loadItem(slug = slug, requestID = requestID)
-  }
-
-  if (webReaderParams != null) {
-    Box(
-      modifier = Modifier
-        .fillMaxSize()
-        .nestedScroll(nestedScrollConnection)
-    ) {
-      Column(
-        modifier = Modifier
-          .fillMaxSize()
-          .verticalScroll(webReaderViewModel.scrollState)
-
-      ) {
-        Row(
-          modifier = Modifier
-            .fillMaxWidth()
-            .requiredHeight(height = maxToolbarHeight)
-        ) {
-        }
-        WebReader(webReaderParams!!, webReaderViewModel.storedWebPreferences(isSystemInDarkTheme()), webReaderViewModel)
-      }
+  Box(
+    modifier = Modifier
+      .fillMaxSize()
+      .systemBarsPadding()
+      .background(color = backgroundColor)
+  ) {
+    if (webReaderParams != null) {
+      WebReader(
+        webReaderParams!!,
+        webReaderViewModel.storedWebPreferences(isSystemInDarkTheme()),
+        webReaderViewModel
+      )
 
       TopAppBar(
         modifier = Modifier
           .height(height = with(LocalDensity.current) {
-            webReaderViewModel.currentToolbarHeight = toolbarHeightPx.value.toInt()
-            toolbarHeightPx.value.roundToInt().toDp()
-          } ),
+            toolbarHeightPx.roundToInt().toDp()
+          }),
         backgroundColor = MaterialTheme.colorScheme.surfaceVariant,
         title = {},
         actions = {
@@ -186,7 +158,12 @@ fun WebReaderLoadingContainer(slug: String? = null, requestID: String? = null, o
             isExpanded = isMenuExpanded,
             isArchived = webReaderParams!!.item.isArchived,
             onDismiss = { isMenuExpanded = false },
-            actionHandler = { webReaderViewModel.handleSavedItemAction(webReaderParams!!.item.savedItemId, it) }
+            actionHandler = {
+              webReaderViewModel.handleSavedItemAction(
+                webReaderParams!!.item.savedItemId,
+                it
+              )
+            }
           )
         }
       )
@@ -218,15 +195,5 @@ fun WebReaderLoadingContainer(slug: String? = null, requestID: String? = null, o
         onBackPressedDispatcher?.onBackPressed()
       }
     }
-  } else {
-    Column(
-      verticalArrangement = Arrangement.SpaceAround,
-      horizontalAlignment = Alignment.CenterHorizontally,
-      modifier = Modifier
-        .fillMaxSize()
-        .padding(horizontal = 16.dp)
-    ) {
-      Text("Loading...", color = Color.White)
-    }
   }
 }

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/reader/WebReaderViewModel.kt

@@ -2,6 +2,10 @@ package app.omnivore.omnivore.ui.reader
 
 import android.util.Log
 import androidx.compose.foundation.ScrollState
+import androidx.compose.runtime.remember
+import androidx.compose.ui.geometry.Offset
+import androidx.compose.ui.input.nestedscroll.NestedScrollConnection
+import androidx.compose.ui.input.nestedscroll.NestedScrollSource
 import androidx.lifecycle.MutableLiveData
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
@@ -34,31 +38,46 @@ class WebReaderViewModel @Inject constructor(
 ): ViewModel() {
   var lastJavascriptActionLoopUUID: UUID = UUID.randomUUID()
   var javascriptDispatchQueue: MutableList<String> = mutableListOf()
-  var scrollState = ScrollState(0)
-  var currentToolbarHeight = 0
+  var maxToolbarHeightPx = 0.0f
 
   val webReaderParamsLiveData = MutableLiveData<WebReaderParams?>(null)
   val annotationLiveData = MutableLiveData<String?>(null)
   val javascriptActionLoopUUIDLiveData = MutableLiveData(lastJavascriptActionLoopUUID)
-  val shouldPopViewLiveData = MutableLiveData<Boolean>(false)
-  val hasFetchError = MutableLiveData<Boolean>(false)
+  val shouldPopViewLiveData = MutableLiveData(false)
+  val hasFetchError = MutableLiveData(false)
+  val currentToolbarHeightLiveData = MutableLiveData(0.0f)
 
   var hasTappedExistingHighlight = false
   var lastTapCoordinates: TapCoordinates? = null
+  private var isLoading = false
   
   fun loadItem(slug: String?, requestID: String?) {
+    if (isLoading || webReaderParamsLiveData.value != null) { return }
+    isLoading = true
+    Log.d("reader", "load item called")
+
     viewModelScope.launch {
       slug?.let { loadItemUsingSlug(it) }
       requestID?.let { loadItemUsingRequestID(it) }
     }
   }
 
+  fun showNavBar() {
+    onScrollChange(maxToolbarHeightPx)
+  }
+
+  fun onScrollChange(delta: Float) {
+    val newHeight = (currentToolbarHeightLiveData.value ?: 0.0f) + delta
+    currentToolbarHeightLiveData.value = newHeight.coerceIn(0f, maxToolbarHeightPx)
+  }
+
   private suspend fun loadItemUsingSlug(slug: String) {
     val webReaderParams = loadItemFromServer(slug)
 
     if (webReaderParams != null) {
-      Log.d("sync", "data loaded from server")
+      Log.d("reader", "data loaded from server")
       webReaderParamsLiveData.postValue(webReaderParams)
+      isLoading = false
     } else {
       loadItemFromDB(slug)
     }
@@ -70,6 +89,7 @@ class WebReaderViewModel @Inject constructor(
 
     if (webReaderParams != null && isSuccessful) {
       webReaderParamsLiveData.postValue(webReaderParams)
+      isLoading = false
     } else if (requestCount < 7) {
       // delay then try again
       delay(2000L)
@@ -96,6 +116,7 @@ class WebReaderViewModel @Inject constructor(
         Log.d("sync", "data loaded from db")
         webReaderParamsLiveData.postValue(WebReaderParams(persistedItem.savedItem, articleContent))
       }
+      isLoading = false
     }
   }
 
@@ -191,16 +212,6 @@ class WebReaderViewModel @Inject constructor(
     }
   }
 
-  fun reset() {
-    shouldPopViewLiveData.postValue(false)
-    webReaderParamsLiveData.value = null
-    annotationLiveData.value = null
-    scrollState = ScrollState(0)
-    javascriptDispatchQueue = mutableListOf()
-    hasTappedExistingHighlight = false
-    lastTapCoordinates = null
-  }
-
   fun resetJavascriptDispatchQueue() {
     lastJavascriptActionLoopUUID = javascriptActionLoopUUIDLiveData.value ?: UUID.randomUUID()
     javascriptDispatchQueue = mutableListOf()

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/root/RootView.kt

@@ -18,14 +18,12 @@ import app.omnivore.omnivore.ui.auth.LoginViewModel
 import app.omnivore.omnivore.ui.auth.WelcomeScreen
 import app.omnivore.omnivore.ui.library.LibraryView
 import app.omnivore.omnivore.ui.library.LibraryViewModel
-import app.omnivore.omnivore.ui.reader.*
 import com.google.accompanist.systemuicontroller.rememberSystemUiController
 
 @Composable
 fun RootView(
   loginViewModel: LoginViewModel,
-  libraryViewModel: LibraryViewModel,
-  webReaderViewModel: WebReaderViewModel
+  libraryViewModel: LibraryViewModel
 ) {
   val hasAuthToken: Boolean by loginViewModel.hasAuthTokenLiveData.observeAsState(false)
   val systemUiController = rememberSystemUiController()
@@ -47,8 +45,7 @@ fun RootView(
     if (hasAuthToken) {
       PrimaryNavigator(
         loginViewModel = loginViewModel,
-        libraryViewModel = libraryViewModel,
-        webReaderViewModel = webReaderViewModel
+        libraryViewModel = libraryViewModel
       )
     } else {
       WelcomeScreen(viewModel = loginViewModel)
@@ -66,8 +63,7 @@ fun RootView(
 @Composable
 fun PrimaryNavigator(
   loginViewModel: LoginViewModel,
-  libraryViewModel: LibraryViewModel,
-  webReaderViewModel: WebReaderViewModel
+  libraryViewModel: LibraryViewModel
 ) {
   val navController = rememberNavController()
 
@@ -79,15 +75,6 @@ fun PrimaryNavigator(
       )
     }
 
-    composable("WebReader/{slug}") {
-      webReaderViewModel.reset() // clear previously loaded item
-
-      WebReaderLoadingContainer(
-        it.arguments?.getString("slug") ?: "",
-        webReaderViewModel = webReaderViewModel
-      )
-    }
-
     composable(Routes.Settings.route) {
       SettingsView(loginViewModel = loginViewModel, navController = navController)
     }

android/Omnivore/app/src/main/java/app/omnivore/omnivore/ui/save/SaveContent.kt

@@ -25,6 +25,7 @@ import kotlinx.coroutines.launch
 fun SaveContent(viewModel: SaveViewModel, modalBottomSheetState: ModalBottomSheetState, modifier: Modifier) {
   val coroutineScope = rememberCoroutineScope()
   val context = LocalContext.current
+  val enableReadNow = false
 
   Surface(modifier = Modifier.fillMaxSize(), color = MaterialTheme.colors.background) {
     Column(
@@ -37,26 +38,28 @@ fun SaveContent(viewModel: SaveViewModel, modalBottomSheetState: ModalBottomShee
     ) {
       Text(text = viewModel.message ?: "Saving")
       Row {
-        Button(
-          onClick = {
-            coroutineScope.launch {
-              modalBottomSheetState.hide()
-              viewModel.clientRequestID?.let {
-                val intent = Intent(context, WebReaderLoadingContainerActivity::class.java)
-                intent.putExtra("SAVED_ITEM_REQUEST_ID", it)
-                context.startActivity(intent)
+        if (enableReadNow) {
+          Button(
+            onClick = {
+              coroutineScope.launch {
+                modalBottomSheetState.hide()
+                viewModel.clientRequestID?.let {
+                  val intent = Intent(context, WebReaderLoadingContainerActivity::class.java)
+                  intent.putExtra("SAVED_ITEM_REQUEST_ID", it)
+                  context.startActivity(intent)
+                }
               }
-            }
-          },
-          colors = ButtonDefaults.buttonColors(
-            contentColor = Color(0xFF3D3D3D),
-            backgroundColor = Color.White
-          )
-        ) {
-          Text(text = "Read Now")
-        }
+            },
+            colors = ButtonDefaults.buttonColors(
+              contentColor = Color(0xFF3D3D3D),
+              backgroundColor = Color.White
+            )
+          ) {
+            Text(text = "Read Now")
+          }
 
-        Spacer(modifier = Modifier.width(8.dp))
+          Spacer(modifier = Modifier.width(8.dp))
+        }
 
         Button(
           onClick = {
@@ -69,10 +72,11 @@ fun SaveContent(viewModel: SaveViewModel, modalBottomSheetState: ModalBottomShee
             backgroundColor = Color(0xffffd234)
           )
         ) {
-          Text(text = "Read Later")
+          Text(text = if (enableReadNow) "Read Later" else "Dismiss")
         }
       }
     }
   }
 }
 
+

apple/OmnivoreKit/Sources/Views/SnoozeView.swift

@@ -21,15 +21,15 @@ public struct SnoozeView: View {
       Spacer()
 
       HStack {
-        SnoozeIconButtonView(snooze: Snooze.snoozeValues[0], action: { snoozeItem($0) })
-        SnoozeIconButtonView(snooze: Snooze.snoozeValues[1], action: { snoozeItem($0) })
+        SnoozeIconButtonView(snooze: Snooze.currentValues[0], action: { snoozeItem($0) })
+        SnoozeIconButtonView(snooze: Snooze.currentValues[1], action: { snoozeItem($0) })
       }
 
       Spacer(minLength: 32)
 
       HStack {
-        SnoozeIconButtonView(snooze: Snooze.snoozeValues[2], action: { snoozeItem($0) })
-        SnoozeIconButtonView(snooze: Snooze.snoozeValues[3], action: { snoozeItem($0) })
+        SnoozeIconButtonView(snooze: Snooze.currentValues[2], action: { snoozeItem($0) })
+        SnoozeIconButtonView(snooze: Snooze.currentValues[3], action: { snoozeItem($0) })
       }
       Spacer()
     }.padding(EdgeInsets(top: 8, leading: 16, bottom: 8, trailing: 16))
@@ -102,14 +102,12 @@ struct Snooze {
     self.untilStr = formatter.string(from: until)
   }
 
-  static var snoozeValues: [Snooze] {
-    let now = Date()
-    return snoozeValuesForDate(now: now)
+  static var currentValues: [Snooze] {
+    calculateValues(for: Date(), calendar: Calendar.current)
   }
 
-  static func snoozeValuesForDate(now: Date) -> [Snooze] {
+  static func calculateValues(for now: Date, calendar: Calendar) -> [Snooze] {
     var res: [Snooze] = []
-    let calendar = Calendar.current
     let components = calendar.dateComponents([.year, .month, .day, .hour, .timeZone, .weekday], from: now)
 
     var tonightComponent = components

apple/OmnivoreKit/Tests/ViewsTests/HomeFeedViewTests.swift

@@ -1,36 +0,0 @@
-@testable import Views
-import XCTest
-
-final class HomeFeedViewTests: XCTestCase {
-  func parse(_ str: String) -> Date {
-    let dateFormatter = DateFormatter()
-    dateFormatter.locale = Locale(identifier: "en_US_POSIX") // set locale to reliable US_POSIX
-    dateFormatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ssZ"
-    return dateFormatter.date(from: str)!
-  }
-
-  func test_weekdayBefore8PM() {
-    let now = parse("2022-01-31T10:11:12-08:00")
-    let snoozes = Snooze.snoozeValuesForDate(now: now)
-
-    XCTAssertEqual(snoozes[0].until, parse("2022-01-31T20:00:00-08:00"))
-    XCTAssertEqual(snoozes[1].until, parse("2022-02-01T08:00:00-08:00"))
-    XCTAssertEqual(snoozes[2].until, parse("2022-02-05T08:00:00-08:00"))
-    XCTAssertEqual(snoozes[3].until, parse("2022-02-07T08:00:00-08:00"))
-  }
-
-  func test_weekdayAfter8PM() {
-    let now = parse("2022-01-31T20:11:12-08:00")
-    let snoozes = Snooze.snoozeValuesForDate(now: now)
-
-    XCTAssertEqual(snoozes[0].until, parse("2022-02-01T08:00:00-08:00"))
-    XCTAssertEqual(snoozes[1].until, parse("2022-02-01T20:00:00-08:00"))
-    XCTAssertEqual(snoozes[2].until, parse("2022-02-05T08:00:00-08:00"))
-    XCTAssertEqual(snoozes[3].until, parse("2022-02-07T08:00:00-08:00"))
-  }
-
-  static var allTests = [
-    ("test_weekdayBefore8PM", test_weekdayBefore8PM),
-    ("test_weekdayAfter8PM", test_weekdayAfter8PM)
-  ]
-}

apple/OmnivoreKit/Tests/ViewsTests/SnoozeTests.swift

@@ -0,0 +1,66 @@
+@testable import Views
+import XCTest
+
+final class SnoozeTests: XCTestCase {
+  func parse(_ str: String) -> Date {
+    let dateFormatter = DateFormatter()
+    dateFormatter.locale = Locale(identifier: "en_US_POSIX") // set locale to reliable US_POSIX
+    dateFormatter.dateFormat = "yyyy-MM-dd'T'HH:mm:ssZ"
+    return dateFormatter.date(from: str)!
+  }
+
+  func test_weekdayBefore8PM_minus8() {
+    var calendar = Calendar(identifier: .gregorian)
+    calendar.timeZone = TimeZone(secondsFromGMT: -8 * 3600)!
+    let now = parse("2022-01-31T10:11:12-08:00")
+    let snoozes = Snooze.calculateValues(for: now, calendar: calendar)
+
+    XCTAssertEqual(snoozes[0].until, parse("2022-01-31T20:00:00-08:00"))
+    XCTAssertEqual(snoozes[1].until, parse("2022-02-01T08:00:00-08:00"))
+    XCTAssertEqual(snoozes[2].until, parse("2022-02-05T08:00:00-08:00"))
+    XCTAssertEqual(snoozes[3].until, parse("2022-02-07T08:00:00-08:00"))
+  }
+
+  func test_weekdayBefore8PM_zulu() {
+    var calendar = Calendar(identifier: .gregorian)
+    calendar.timeZone = TimeZone(secondsFromGMT: 0)!
+    let now = parse("2022-01-31T13:14:15Z")
+    let snoozes = Snooze.calculateValues(for: now, calendar: calendar)
+
+    XCTAssertEqual(snoozes[0].until, parse("2022-01-31T20:00:00Z"))
+    XCTAssertEqual(snoozes[1].until, parse("2022-02-01T08:00:00Z"))
+    XCTAssertEqual(snoozes[2].until, parse("2022-02-05T08:00:00Z"))
+    XCTAssertEqual(snoozes[3].until, parse("2022-02-07T08:00:00Z"))
+  }
+
+  func test_weekdayAfter8PM_minus8() {
+    var calendar = Calendar(identifier: .gregorian)
+    calendar.timeZone = TimeZone(secondsFromGMT: -8 * 3600)!
+    let now = parse("2022-01-31T20:11:12-08:00")
+    let snoozes = Snooze.calculateValues(for: now, calendar: calendar)
+
+    XCTAssertEqual(snoozes[0].until, parse("2022-02-01T08:00:00-08:00"))
+    XCTAssertEqual(snoozes[1].until, parse("2022-02-01T20:00:00-08:00"))
+    XCTAssertEqual(snoozes[2].until, parse("2022-02-05T08:00:00-08:00"))
+    XCTAssertEqual(snoozes[3].until, parse("2022-02-07T08:00:00-08:00"))
+  }
+
+  func test_weekdayAfter8PM_plus5() {
+    var calendar = Calendar(identifier: .gregorian)
+    calendar.timeZone = TimeZone(secondsFromGMT: 5 * 3600)!
+    let now = parse("2022-01-31T22:33:44+05:00")
+    let snoozes = Snooze.calculateValues(for: now, calendar: calendar)
+
+    XCTAssertEqual(snoozes[0].until, parse("2022-02-01T08:00:00+05:00"))
+    XCTAssertEqual(snoozes[1].until, parse("2022-02-01T20:00:00+05:00"))
+    XCTAssertEqual(snoozes[2].until, parse("2022-02-05T08:00:00+05:00"))
+    XCTAssertEqual(snoozes[3].until, parse("2022-02-07T08:00:00+05:00"))
+  }
+
+  static var allTests = [
+    ("test_weekdayBefore8PM_minus8", test_weekdayBefore8PM_minus8),
+    ("test_weekdayBefore8PM_zulu", test_weekdayBefore8PM_zulu),
+    ("test_weekdayAfter8PM_minus8", test_weekdayAfter8PM_minus8),
+    ("test_weekdayAfter8PM_plus5", test_weekdayAfter8PM_plus5)
+  ]
+}

docs/guides/getting-started/fr/getting-started-guide.md

@@ -0,0 +1,172 @@
+# Commencer avec Omnivore
+
+Omnivore est une **application de lecture ultérieure** qui vous permet de sauvegarder et d'organiser tout ce que vous lisez en ligne.
+
+Ce guide vous montrera comment utiliser les fonctions de base et les fonctionnalités avancées d'Omnivore, divisées en quatre activités principales:
+
+
+* Sauvegarde
+* Lecture
+* Organisation
+* Intégrations
+
+La **Library** est le centre de votre expérience Omnivore, où vous pouvez accéder rapidement à tous les liens que vous avez enregistrés. Les liens enregistrés restent dans votre bibliothèque à jamais, à moins que vous ne les supprimiez.
+
+
+## Sauvegarde
+
+
+Il existe cinq façons de sauvegarder des liens vers des pages ou des articles que vous souhaitez lire plus tard:
+
+
+
+* Sauvegarde depuis votre bibliothèque Omnivore
+* Sauvegarde depuis un navigateur
+* Sauvegarde depuis un téléphone ou une tablette (iOS ou Android)
+* Abonnements à des newsletters via e-mail
+* Sauvegarde de PDF à partir d'un Mac
+
+
+### Sauvegarde depuis votre bibliothèque Omnivore
+1. Dans le coin supérieur droit de votre bibliothèque, appuyez sur le bouton **Add Link**.
+2. Entrez l'URL que vous souhaitez enregistrer et appuyez sur **Add Link**.
+3. Le lien apparaîtra dans votre bibliothèque la prochaine fois que vous la rafraîchissez.
+
+
+### Sauvegarde depuis un navigateur
+
+1. Téléchargez et installez l'extension Omnivore pour votre navigateur :
+
+* [Chrome ](https://omnivore.app/install/chrome)
+* [Edge](https://omnivore.app/install/edge)
+* [Firefox](https://omnivore.app/install/firefox)
+* [Safari](https://omnivore.app/install/safari)
+
+2. Accédez à la page que vous souhaitez enregistrer et appuyez sur le bouton Omnivore dans la barre d'outils de votre navigateur ou dans le menu Extensions.
+3. Alternativement, vous pouvez cliquer avec le bouton droit (commande + clic sur Mac) sur n'importe quel lien hypertexte et sélectionner **Enregistrer dans Omnivore** dans le menu.
+4. Le lien apparaîtra dans votre bibliothèque la prochaine fois que vous la rafraîchissez.
+
+
+
+### Sauvegarde depuis un téléphone ou une tablette
+
+La meilleure façon d'enregistrer des liens à partir de votre appareil mobile est via l'application Omnivore. Vous pouvez télécharger l'application ici :
+
+* [iOS (iPhone ou iPad)](https://omnivore.app/install/ios)
+* Android
+
+Une fois que l'application mobile est installée :
+
+1. Dans votre navigateur, accédez à la page que vous souhaitez enregistrer et appuyez sur le bouton **Share**.
+2. Appuyez sur l'icône **Omnivore** dans le menu Partager.
+3. Le lien apparaîtra dans votre bibliothèque la prochaine fois que vous la rafraîchirez.
+
+### Abonnements à la newsletter via e-mail
+
+1. Sur le site web ou l'application Omnivore, appuyez sur votre photo, initiale ou avatar dans le coin supérieur droit pour accéder au menu de profil. Sélectionnez **Emails** dans le menu.
+2. Appuyez sur **Create a New Email Address** pour ajouter une nouvelle adresse e-mail (par exemple : username-123_abc@inbox.omnivore.app) à la liste.
+3. Cliquez sur l'icône Copier à côté de l'adresse e-mail.
+4. Accédez à la page d'inscription pour la newsletter à laquelle vous souhaitez vous abonner.
+5. Collez l'adresse e-mail Omnivore dans le formulaire d'inscription.
+6. Les nouvelles newsletters seront automatiquement livrées à votre boîte de réception Omnivore.
+
+
+### Enregistrement de PDF à partir d'un Mac
+
+1. Installez [l'application Mac](https://omnivore.app/install/mac). 
+2. Sur votre Mac, localisez le PDF que vous souhaitez enregistrer et faites un clic droit ou ctrl + clic sur le nom de fichier.
+3. Sélectionnez **Share** dans le menu et choisissez **Omnivore**.
+4. Le lien apparaîtra dans votre bibliothèque la prochaine fois que vous la rafraîchirez.
+
+
+## Lecture
+
+Cliquez sur n'importe quel lien enregistré dans votre bibliothèque pour accéder à la vue du lecteur. 
+
+Omnivore formatte les pages pour une lecture facile et une surbrillance, en enlevant les publicités et les encombrements pour une lecture exempte de distractions. La vue axée sur le texte rend également les articles plus petits et plus rapides à charger.
+
+Pendant la lecture, vous pouvez :
+
+* <span style="text-decoration:underline;">Modifier la mise en forme</span>
+* <span style="text-decoration:underline;">Surligner le texte</span>
+* <span style="text-decoration:underline;">Ajouter des notes</span>
+* <span style="text-decoration:underline;">Voir toutes les surlignages et notes enregistrés</span>
+* <span style="text-decoration:underline;">Suivre la progression de la lecture</span>
+
+### Modification de la mise en forme
+
+1. **_Thème:_** Appuyez sur votre photo, initiale ou avatar dans le coin supérieur droit pour accéder au menu de profil. Sélectionnez la miniature blanche ou noire pour choisir le thème clair ou sombre.
+2. **_Mise en forme du texte:_** Appuyez sur l'icône Aa pour ajuster la taille du texte, la police, les marges et les espacements entre les lignes.
+
+### Mettre en surbrillance du texte
+
+1. Sélectionnez le texte que vous souhaitez mettre en surbrillance.
+2. Appuyez sur le bouton **Highlight**.
+3. Le texte apparaîtra en surbrillance la prochaine fois que vous consulterez l'article.
+
+### Ajouter des notes
+
+1. Mettre en surbrillance une section de texte où vous souhaitez ajouter une note.
+2. Appuyez sur le bouton **Note**, tapez votre note et appuyez sur **Save**.
+3. L'icône de note apparaîtra la prochaine fois que vous consulterez cet article.
+
+### Afficher toutes les surbrillances et les notes enregistrées
+
+1. Appuyez sur l'icône Highlight/Note pour afficher une liste de toutes les sections de texte mises en surbrillance et les notes que vous avez ajoutées à cette page.
+2. Pour supprimer une note ou une surbrillance, sélectionnez-la dans la liste et appuyez sur l'icône Corbeille.
+
+### Suivre l'avancement de la lecture
+
+Omnivore garde automatiquement une trace de votre avancement de lecture sur vos différents appareils, de sorte que vous puissiez facilement reprendre là où vous vous êtes arrêté. Une barre de progression apparaîtra en haut de chaque lien de votre bibliothèque après que vous avez commencé à lire.
+
+## Organisation
+
+Par défaut, la boîte de réception de la bibliothèque affiche tous les liens que vous avez enregistrés. Pour gérer votre liste et maintenir votre lecture organisée, Omnivore fournit les actions suivantes:
+
+* <span style="text-decoration:underline;">Archiving</span>
+* <span style="text-decoration:underline;">Labels</span>
+* <span style="text-decoration:underline;">Search</span>
+* <span style="text-decoration:underline;">Filters</span>
+
+### Archivage
+
+1. Appuyez sur l'icône Menu à côté du lien que vous souhaitez archiver (dans l'application mobile, appuyez longuement sur le lien pour ouvrir le menu).
+2. Sélectionnez **Archive**.
+3. Le lien disparaîtra de la vue par défaut de la bibliothèque, mais sera visible si vous sélectionnez le filtre Archivé (voir <span style="text-decoration:underline;">Filtres </span>ci-dessous).
+
+### Étiquettes
+
+1. Appuyez sur l'icône Menu à côté de n'importe quel lien et sélectionnez **Set Labels**.
+2. Sélectionnez une étiquette existante dans la liste ou appuyez sur **Edit Labels** les étiquettes pour en créer une nouvelle.
+3. L'étiquette apparaîtra à côté du lien dans votre bibliothèque. Appuyez dessus pour afficher tous les liens avec la même étiquette.
+4. _Application mobile d'Omnivore uniquement_: appuyez sur **Labels ** pour voir une liste complète de toutes les étiquettes que vous avez utilisées; appuyez sur une étiquette pour afficher tous les liens avec la même étiquette.
+5. Note : Omnivore attribuera automatiquement certaines étiquettes, telles que "Newsletters".
+
+### Recherche
+
+1. Pour rechercher parmi tous les liens que vous avez enregistrés, entrez un mot-clé ou une phrase dans la barre de recherche. 
+2. Vous pouvez combiner les mots-clés avec les étiquettes et les filtres pour cibler encore plus votre recherche. [En savoir plus sur la recherche avancée.](https://omnivore.app/help/search).
+
+### Filtres
+
+1. Utilisez le menu **Filters** pour affiner votre vue de la bibliothèque (certains filtres peuvent être visibles par défaut).
+2. Sélectionnez **Read Later** pour afficher une liste de tous vos liens non archivés à l'exception des newsletters.
+3. Sélectionnez **Highlights** pour afficher les sélections de texte que vous avez surlignées dans toutes les pages que vous avez enregistrées.
+4. Sélectionnez **Today** pour afficher une liste de liens que vous avez enregistrés aujourd'hui.
+5. Sélectionnez **Newsletters** pour afficher les liens enregistrés via vos abonnements à des newsletters.
+
+## Intégrations
+
+Omnivore permet des intégrations avec des bases de connaissances et des applications de prise de notes, notamment :
+
+* Logseq
+* Webhooks
+
+### Logseq
+
+Avec le plugin Omnivore pour Logseq, vous pouvez synchroniser tous vos articles enregistrés, vos surlignages et vos notes dans Logseq, une base de connaissances populaire. Pour obtenir des informations sur la configuration et l'utilisation du plugin Logseq, veuillez consulter ce guide utile [Omnivore pour le plugin Logseq](https://briansunter.com/graph/#/page/omnivore-logseq-guide).
+
+
+### Webhooks
+
+Omnivore peut déclencher des webhooks lorsque vous enregistrez un lien ou ajoutez des surlignages à une page que vous lisez. <span style="text-decoration:underline;">Cet exemple</span> montre comment utiliser les webhooks pour écrire tous les liens enregistrés dans une feuille de calcul Google Sheets stockée sur Google Drive.

package.json

@@ -17,12 +17,12 @@
     "deploy:web": "vercel --prod"
   },
   "devDependencies": {
+    "@ardatan/aggregate-error": "^0.0.6",
     "@graphql-codegen/cli": "^2.6.2",
     "@graphql-codegen/introspection": "^2.1.1",
     "@graphql-codegen/schema-ast": "^2.1.1",
     "@graphql-codegen/typescript": "^2.1.1",
     "@graphql-codegen/typescript-resolvers": "^2.1.1",
-    "@ardatan/aggregate-error": "^0.0.6",
     "@tsconfig/node14": "^1.0.1",
     "@typescript-eslint/eslint-plugin": "^5.9.0",
     "@typescript-eslint/parser": "^5.9.0",
@@ -38,5 +38,6 @@
   "volta": {
     "node": "14.18.1",
     "yarn": "1.22.19"
-  }
+  },
+  "dependencies": {}
 }

packages/api/apollo.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  service: {
+    localSchemaFile: './src/generated/schema.graphql',
+  },
+}

packages/api/package.json

@@ -26,7 +26,7 @@
     "@opentelemetry/api": "^1.0.1",
     "@opentelemetry/core": "^1.3.1",
     "@opentelemetry/exporter-jaeger": "^1.0.1",
-    "@opentelemetry/instrumentation-dns": "^0.29.0",
+    "@opentelemetry/instrumentation-dns": "^0.31.2",
     "@opentelemetry/instrumentation-express": "^0.28.0",
     "@opentelemetry/instrumentation-graphql": "^0.29.0",
     "@opentelemetry/instrumentation-grpc": "^0.29.2",
@@ -54,10 +54,11 @@
     "dompurify": "^2.0.17",
     "dot-case": "^3.0.4",
     "dotenv": "^8.2.0",
+    "elastic-ts": "^0.9.0",
     "express": "^4.17.1",
     "express-http-context": "^1.2.4",
     "express-rate-limit": "^6.3.0",
-    "firebase-admin": "^10.0.2",
+    "firebase-admin": "^11.5.0",
     "googleapis": "^105.0.0",
     "graphql": "^15.3.0",
     "graphql-fields": "^2.0.3",
@@ -68,7 +69,7 @@
     "intercom-client": "^3.1.4",
     "jsonwebtoken": "^8.5.1",
     "jwks-rsa": "^2.0.3",
-    "knex": "0.21.12",
+    "knex": "2.4.2",
     "knex-stringcase": "^1.4.2",
     "linkedom": "^0.14.9",
     "luxon": "^3.2.1",

packages/api/src/apollo.ts

@@ -7,7 +7,7 @@
 import { ContextFunction } from 'apollo-server-core'
 import { ClaimsToSet, ResolverContext } from './resolvers/types'
 import { SetClaimsRole } from './utils/dictionary'
-import Knex, { Transaction } from 'knex'
+import { Knex } from 'knex'
 import { ExpressContext } from 'apollo-server-express/dist/ApolloServer'
 import * as jwt from 'jsonwebtoken'
 import { kx } from './datalayer/knex_config'
@@ -48,7 +48,7 @@ const contextFunc: ContextFunction<ExpressContext, ResolverContext> = async ({
   const claims = await getClaimsByToken(token)
 
   async function setClaims(
-    tx: Transaction,
+    tx: Knex.Transaction,
     uuid?: string,
     userRole?: string
   ): Promise<void> {

packages/api/src/datalayer/article/index.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import { ArticleData, CreateSet, keys as modelKeys, UpdateSet } from './model'
 import DataModel from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { Table } from '../../utils/dictionary'
 import { logMethod } from '../helpers'
 

packages/api/src/datalayer/article_saving_request/index.ts

@@ -1,11 +1,11 @@
 import {
+  ArticleSavingRequestData,
   CreateSet,
   keys as modelKeys,
   UpdateSet,
-  ArticleSavingRequestData,
 } from './model'
 import DataModel from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { Table } from '../../utils/dictionary'
 import { logMethod } from '../helpers'
 

packages/api/src/datalayer/helpers.ts

@@ -5,7 +5,7 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import DataModel from './model'
-import Knex, { Transaction } from 'knex'
+import { Knex } from 'knex'
 import DataLoader from 'dataloader'
 import { snakeCase } from 'snake-case'
 import { buildLogger } from '../utils/logger'
@@ -14,7 +14,7 @@ import { SetClaimsRole } from '../utils/dictionary'
 const logger = buildLogger('datalayer')
 
 export const setClaims = async (
-  tx: Transaction,
+  tx: Knex.Transaction,
   uuid?: string,
   userRole?: string
 ): Promise<void> => {

packages/api/src/datalayer/highlight/index.ts

@@ -4,7 +4,7 @@
 import DataModel, { DataModelError, MAX_RECORDS_LIMIT } from '../model'
 import { CreateSet, HighlightData, keys as modelKeys, UpdateSet } from './model'
 import { Table } from '../../utils/dictionary'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { ENABLE_DB_REQUEST_LOGGING, globalCounter } from '../helpers'
 import DataLoader from 'dataloader'
 
@@ -26,7 +26,8 @@ class HighlightModel extends DataModel<HighlightData, CreateSet, UpdateSet> {
         }
 
         try {
-          const rows: HighlightData[] = await kx(this.tableName)
+          const rows: HighlightData[] = await kx
+            .table(this.tableName)
             .select(this.modelKeys)
             .whereIn('id', keys)
             .andWhere('deleted', false)
@@ -60,7 +61,8 @@ class HighlightModel extends DataModel<HighlightData, CreateSet, UpdateSet> {
         )
       }
 
-      const result = await this.kx(Table.HIGHLIGHT)
+      const result = await this.kx
+        .table(Table.HIGHLIGHT)
         .select(modelKeys)
         .whereIn('elasticPageId', articleIds)
         .andWhere('deleted', false)
@@ -153,7 +155,8 @@ class HighlightModel extends DataModel<HighlightData, CreateSet, UpdateSet> {
     userId: string,
     articleId: string
   ): Promise<HighlightData[]> {
-    const highlights = await this.kx(Table.HIGHLIGHT)
+    const highlights: HighlightData[] = await this.kx
+      .table(Table.HIGHLIGHT)
       .select(modelKeys)
       .where('user_id', userId)
       .andWhere('elastic_page_id', articleId)

packages/api/src/datalayer/knex_config.ts

@@ -1,8 +1,8 @@
 import { env } from '../env'
-import Knex from 'knex'
+import knex from 'knex'
 import knexStringcase from 'knex-stringcase'
 
-export const kx = Knex(
+export const kx = knex(
   knexStringcase({
     client: 'pg',
     connection: {

packages/api/src/datalayer/links/index.ts

@@ -12,7 +12,7 @@ import {
   UserFeedArticleData,
 } from './model'
 import DataModel, { MAX_RECORDS_LIMIT } from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { Table } from '../../utils/dictionary'
 import {
   Article,

packages/api/src/datalayer/links/share_info.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-unsafe-return */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { LinkShareInfo } from '../../generated/graphql'
 import { DataModels } from '../../resolvers/types'
 import { getPageByParam } from '../../elastic/pages'

packages/api/src/datalayer/model.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-floating-promises */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import DataLoader from 'dataloader'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { ENABLE_DB_REQUEST_LOGGING, globalCounter } from './helpers'
 
 export enum DataModelError {
@@ -113,7 +113,7 @@ abstract class DataModel<
     tx: Knex.Transaction
   ): Promise<ModelData[]> {
     const rows: ModelData[] = await tx
-      .batchInsert(this.tableName, data)
+      .batchInsert(this.tableName, data as never)
       .returning('*')
     for (const row of rows) {
       this.loader.prime(row.id, row)

packages/api/src/datalayer/reaction/index.ts

@@ -1,11 +1,11 @@
 import {
-  globalCounter,
   ENABLE_DB_REQUEST_LOGGING,
+  globalCounter,
   logMethod,
 } from './../helpers'
-import { CreateSet, keys as modelKeys, UpdateSet, ReactionData } from './model'
+import { CreateSet, keys as modelKeys, ReactionData, UpdateSet } from './model'
 import DataModel, { DataModelError, MAX_RECORDS_LIMIT } from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { Table } from '../../utils/dictionary'
 
 import DataLoader from 'dataloader'

packages/api/src/datalayer/reminders/index.ts

@@ -6,7 +6,7 @@ import {
   UpdateSet,
 } from './model'
 import DataModel, { DataModelError } from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { Table } from '../../utils/dictionary'
 import { logMethod } from '../helpers'
 import { ArticleData } from '../article/model'

packages/api/src/datalayer/report/index.ts

@@ -1,4 +1,4 @@
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { ReportType } from '../../generated/graphql'
 
 interface ReportItem {

packages/api/src/datalayer/task/index.ts

@@ -1,6 +1,6 @@
-import { CreateSet, keys as modelKeys, UpdateSet, TaskData } from './model'
+import { CreateSet, keys as modelKeys, TaskData, UpdateSet } from './model'
 import DataModel from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { logMethod } from '../helpers'
 
 class TaskModel extends DataModel<TaskData, CreateSet, UpdateSet> {

packages/api/src/datalayer/upload_files/index.ts

@@ -7,7 +7,7 @@ import {
   UploadFileData,
 } from './model'
 import DataModel from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { Table } from '../../utils/dictionary'
 import { logMethod } from '../helpers'
 

packages/api/src/datalayer/user/index.ts

@@ -11,7 +11,7 @@ import {
   UserData,
 } from './model'
 import DataModel, { DataModelError, MAX_RECORDS_LIMIT } from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { ENABLE_DB_REQUEST_LOGGING, globalCounter, logMethod } from '../helpers'
 import { Table } from '../../utils/dictionary'
 import DataLoader from 'dataloader'

packages/api/src/datalayer/user_friends/index.ts

@@ -6,7 +6,7 @@ import {
   UserFriendData,
 } from './model'
 import DataModel, { MAX_RECORDS_LIMIT } from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { Table } from '../../utils/dictionary'
 import { ENABLE_DB_REQUEST_LOGGING, globalCounter, logMethod } from '../helpers'
 import DataLoader from 'dataloader'

packages/api/src/datalayer/user_personalization/index.ts

@@ -9,7 +9,7 @@ import {
   UserPersonalizationData,
 } from './model'
 import DataModel from '../model'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { Table } from '../../utils/dictionary'
 import { camelCase } from 'voca'
 import { logMethod } from '../helpers'

packages/api/src/directives.ts

@@ -15,6 +15,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => {
       }
 
       const maxLength = sanitizeDirective.maxLength as number | undefined
+      const minLength = sanitizeDirective.minLength as number | undefined
       const allowedTags = sanitizeDirective.allowedTags as string[] | undefined
       const pattern = sanitizeDirective.pattern as string | undefined
 
@@ -27,6 +28,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => {
             fieldConfig.type.ofType,
             allowedTags,
             maxLength,
+            minLength,
             pattern
           )
         )
@@ -35,6 +37,7 @@ export const sanitizeDirectiveTransformer = (schema: GraphQLSchema) => {
           fieldConfig.type,
           allowedTags,
           maxLength,
+          minLength,
           pattern
         )
       } else {

packages/api/src/elastic/highlights.ts

@@ -29,7 +29,7 @@ export const addHighlightToPage = async (
                   ctx._source.updatedAt = params.highlight.updatedAt`,
           lang: 'painless',
           params: {
-            highlight: highlight,
+            highlight,
           },
         },
       },
@@ -136,9 +136,12 @@ export const deleteHighlight = async (
       refresh: ctx.refresh,
     })
 
-    if (body.updated === 0) return false
-
-    await ctx.pubsub.entityDeleted(EntityType.HIGHLIGHT, highlightId, ctx.uid)
+    body.updated > 0 &&
+      (await ctx.pubsub.entityDeleted(
+        EntityType.HIGHLIGHT,
+        highlightId,
+        ctx.uid
+      ))
 
     return true
   } catch (e) {
@@ -263,7 +266,7 @@ export const updateHighlight = async (
                    ctx._source.updatedAt = params.highlight.updatedAt`,
           lang: 'painless',
           params: {
-            highlight: highlight,
+            highlight,
           },
         },
         query: {
@@ -289,15 +292,15 @@ export const updateHighlight = async (
         },
       },
       refresh: ctx.refresh,
+      conflicts: 'proceed',
     })
 
-    if (body.updated === 0) return false
-
-    await ctx.pubsub.entityUpdated<Highlight>(
-      EntityType.HIGHLIGHT,
-      highlight,
-      ctx.uid
-    )
+    body.updated > 0 &&
+      (await ctx.pubsub.entityUpdated<Highlight>(
+        EntityType.HIGHLIGHT,
+        highlight,
+        ctx.uid
+      ))
 
     return true
   } catch (e) {

packages/api/src/elastic/labels.ts

@@ -57,7 +57,8 @@ export const addLabelInPage = async (
 export const updateLabelsInPage = async (
   pageId: string,
   labels: Label[],
-  ctx: PageContext
+  ctx: PageContext,
+  labelsToAdd?: Label[]
 ): Promise<boolean> => {
   try {
     const { body } = await client.update({
@@ -74,12 +75,16 @@ export const updateLabelsInPage = async (
     })
 
     if (body.result !== 'updated') return false
-
-    for (const label of labels) {
-      await ctx.pubsub.entityCreated<Label & { pageId: string }>(
-        EntityType.LABEL,
-        { pageId, ...label },
-        ctx.uid
+    if (labelsToAdd) {
+      // publish labels to be added
+      await Promise.all(
+        labelsToAdd.map((label) =>
+          ctx.pubsub.entityCreated<Label & { pageId: string }>(
+            EntityType.LABEL,
+            { pageId, ...label },
+            ctx.uid
+          )
+        )
       )
     }
 
@@ -277,7 +282,7 @@ export const setLabelsForHighlight = async (
           lang: 'painless',
           params: {
             highlightId,
-            labels: labels,
+            labels,
             updatedAt: new Date(),
           },
         },

packages/api/src/elastic/pages.ts

@@ -1,13 +1,7 @@
-import {
-  ArticleSavingRequestStatus,
-  Page,
-  PageContext,
-  PageSearchArgs,
-  PageType,
-  ParamSet,
-  SearchBody,
-  SearchResponse,
-} from './types'
+import { ResponseError } from '@elastic/elasticsearch/lib/errors'
+import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts'
+import { EntityType } from '../datalayer/pubsub'
+import { BulkActionType } from '../generated/graphql'
 import {
   DateFilter,
   FieldFilter,
@@ -15,179 +9,173 @@ import {
   InFilter,
   LabelFilter,
   LabelFilterType,
+  NoFilter,
   ReadFilter,
   SortBy,
   SortOrder,
 } from '../utils/search'
 import { client, INDEX_ALIAS } from './index'
-import { EntityType } from '../datalayer/pubsub'
-import { ResponseError } from '@elastic/elasticsearch/lib/errors'
-import { BulkActionType } from '../generated/graphql'
+import {
+  ArticleSavingRequestStatus,
+  Page,
+  PageContext,
+  PageSearchArgs,
+  PageType,
+  ParamSet,
+  SearchResponse,
+} from './types'
 
-const appendQuery = (body: SearchBody, query: string): void => {
-  body.query.bool.should.push({
-    multi_match: {
+const appendQuery = (builder: ESBuilder, query: string): ESBuilder => {
+  return builder
+    .orQuery('multi_match', {
       query,
       fields: ['title', 'content', 'author', 'description', 'siteName'],
       operator: 'and',
       type: 'cross_fields',
-    },
-  })
-  body.query.bool.minimum_should_match = 1
+    })
+    .queryMinimumShouldMatch(1)
 }
 
-const appendTypeFilter = (body: SearchBody, filter: PageType): void => {
-  body.query.bool.must.push({
-    term: {
-      pageType: filter,
-    },
-  })
+const appendTypeFilter = (builder: ESBuilder, filter: PageType): ESBuilder => {
+  return builder.query('term', { pageType: filter })
 }
 
-const appendReadFilter = (body: SearchBody, filter: ReadFilter): void => {
+const appendReadFilter = (
+  builder: ESBuilder,
+  filter: ReadFilter
+): ESBuilder => {
   switch (filter) {
     case ReadFilter.UNREAD:
-      body.query.bool.must.push({
-        range: {
-          readingProgressPercent: {
-            lt: 98,
-          },
+      return builder.query('range', {
+        readingProgressPercent: {
+          lt: 98,
         },
       })
-      break
     case ReadFilter.READ:
-      body.query.bool.must.push({
-        range: {
-          readingProgressPercent: {
-            gte: 98,
-          },
+      return builder.query('range', {
+        readingProgressPercent: {
+          gte: 98,
         },
       })
   }
+  return builder
 }
 
-const appendInFilter = (body: SearchBody, filter: InFilter): void => {
+const appendInFilter = (builder: ESBuilder, filter: InFilter): ESBuilder => {
   switch (filter) {
     case InFilter.ARCHIVE:
-      body.query.bool.must.push({
-        exists: {
-          field: 'archivedAt',
-        },
-      })
-      break
+      return builder.query('exists', { field: 'archivedAt' })
     case InFilter.INBOX:
-      body.query.bool.must_not.push({
-        exists: {
-          field: 'archivedAt',
-        },
-      })
+      return builder.notQuery('exists', { field: 'archivedAt' })
   }
+  return builder
 }
 
-const appendHasFilters = (body: SearchBody, filters: HasFilter[]): void => {
+const appendHasFilters = (
+  builder: ESBuilder,
+  filters: HasFilter[]
+): ESBuilder => {
   filters.forEach((filter) => {
     switch (filter) {
       case HasFilter.HIGHLIGHTS:
-        body.query.bool.must.push({
-          nested: {
-            path: 'highlights',
-            query: {
-              exists: {
-                field: 'highlights',
-              },
+        builder = builder.query('nested', {
+          path: 'highlights',
+          query: {
+            exists: {
+              field: 'highlights',
             },
           },
         })
         break
       case HasFilter.SHARED_AT:
-        body.query.bool.must.push({
-          exists: {
-            field: 'sharedAt',
-          },
-        })
+        builder = builder.query('exists', { field: 'sharedAt' })
         break
     }
   })
+  return builder
 }
 
 const appendExcludeLabelFilter = (
-  body: SearchBody,
+  builder: ESBuilder,
   filters: LabelFilter[]
-): void => {
-  body.query.bool.must_not.push({
-    nested: {
-      path: 'labels',
-      query: filters.map((filter) => {
-        return {
-          terms: {
-            'labels.name': filter.labels,
-          },
-        }
-      }),
+): ESBuilder => {
+  const labels = filters.map((filter) => filter.labels).flat()
+  return builder.notQuery('nested', {
+    path: 'labels',
+    query: {
+      terms: {
+        'labels.name': labels,
+      },
     },
   })
 }
 
 const appendIncludeLabelFilter = (
-  body: SearchBody,
+  builder: ESBuilder,
   filters: LabelFilter[]
-): void => {
+): ESBuilder => {
   filters.forEach((filter) => {
-    body.query.bool.must.push({
-      nested: {
-        path: 'labels',
-        query: {
-          terms: {
-            'labels.name': filter.labels,
-          },
+    builder = builder.query('nested', {
+      path: 'labels',
+      query: {
+        terms: {
+          'labels.name': filter.labels,
         },
       },
     })
   })
+  return builder
 }
 
-const appendDateFilters = (body: SearchBody, filters: DateFilter[]): void => {
+const appendDateFilters = (
+  builder: ESBuilder,
+  filters: DateFilter[]
+): ESBuilder => {
   filters.forEach((filter) => {
-    body.query.bool.must.push({
-      range: {
-        [filter.field]: {
-          gt: filter.startDate,
-          lt: filter.endDate,
-        },
+    builder = builder.query('range', {
+      [filter.field]: {
+        gt: filter.startDate?.getTime(),
+        lt: filter.endDate?.getTime(),
       },
     })
   })
+  return builder
 }
 
-const appendTermFilters = (body: SearchBody, filters: FieldFilter[]): void => {
+const appendTermFilters = (
+  builder: ESBuilder,
+  filters: FieldFilter[]
+): ESBuilder => {
   filters.forEach((filter) => {
-    body.query.bool.must.push({
-      term: {
-        [filter.field]: filter.value,
-      },
+    builder = builder.query('term', {
+      [filter.field]: filter.value,
     })
   })
+  return builder
 }
 
-const appendMatchFilters = (body: SearchBody, filters: FieldFilter[]): void => {
+const appendMatchFilters = (
+  builder: ESBuilder,
+  filters: FieldFilter[]
+): ESBuilder => {
   filters.forEach((filter) => {
-    body.query.bool.must.push({
-      match: {
-        [filter.field]: filter.value,
-      },
+    builder = builder.query('match', {
+      [filter.field]: filter.value,
     })
   })
+  return builder
 }
 
-const appendIdsFilter = (body: SearchBody, ids: string[]): void => {
-  body.query.bool.must.push({
-    terms: {
-      _id: ids,
-    },
+const appendIdsFilter = (builder: ESBuilder, ids: string[]): ESBuilder => {
+  return builder.query('terms', {
+    _id: ids,
   })
 }
 
-const appendRecommendedBy = (body: SearchBody, recommendedBy: string): void => {
+const appendRecommendedBy = (
+  builder: ESBuilder,
+  recommendedBy: string
+): ESBuilder => {
   const query =
     recommendedBy === '*'
       ? {
@@ -200,12 +188,48 @@ const appendRecommendedBy = (body: SearchBody, recommendedBy: string): void => {
             'recommendations.name': recommendedBy,
           },
         }
+  return builder.query('nested', {
+    path: 'recommendations',
+    query,
+  })
+}
 
-  body.query.bool.must.push({
-    nested: {
-      path: 'recommendations',
-      query,
-    },
+const appendNoFilters = (
+  builder: ESBuilder,
+  noFilters: NoFilter[]
+): ESBuilder => {
+  noFilters.forEach((filter) => {
+    builder = builder.notQuery('nested', {
+      path: filter.field,
+      query: {
+        exists: {
+          field: filter.field,
+        },
+      },
+    })
+  })
+  return builder
+}
+
+const appendSiteNameFilter = (
+  builder: ESBuilder,
+  siteName: string
+): ESBuilder => {
+  return builder.query('bool', {
+    should: [
+      {
+        match: {
+          siteName,
+        },
+      },
+      {
+        wildcard: {
+          // siteName is a domain name, so we need to wildcard the end
+          url: `*${siteName}*`,
+        },
+      },
+    ],
+    minimum_should_match: 1,
   })
 }
 
@@ -241,7 +265,7 @@ export const updatePage = async (
   ctx: PageContext
 ): Promise<boolean> => {
   try {
-    const { body } = await client.update({
+    await client.update({
       index: INDEX_ALIAS,
       id,
       body: {
@@ -254,8 +278,6 @@ export const updatePage = async (
       retry_on_conflict: 3,
     })
 
-    if (body.result !== 'updated') return false
-
     if (page.state === ArticleSavingRequestStatus.Deleted) {
       await ctx.pubsub.entityDeleted(EntityType.PAGE, id, ctx.uid)
       return true
@@ -306,31 +328,30 @@ export const deletePage = async (
 }
 
 export const getPageByParam = async <K extends keyof ParamSet>(
-  param: Record<K, ParamSet[K]>,
+  params: Record<K, ParamSet[K] | ParamSet[K][]>,
   includeOriginalHtml = false
 ): Promise<Page | undefined> => {
   try {
-    const params = {
-      query: {
-        bool: {
-          filter: Object.keys(param).map((key) => {
-            return {
-              term: {
-                [key]: param[key as K],
-              },
-            }
-          }),
-        },
-      },
-      size: 1,
-      _source: {
+    let builder = esBuilder()
+      .size(1)
+      .rawOption('_source', {
         excludes: includeOriginalHtml ? [] : ['originalHtml'],
-      },
-    }
-
+      })
+    // filter out undefined and null values and empty arrays
+    // and build the query
+    Object.entries<ParamSet[K] | ParamSet[K][]>(params)
+      .filter(
+        ([, value]) =>
+          value != null && !(Array.isArray(value) && value.length === 0)
+      )
+      .forEach(([key, value]) => {
+        Array.isArray(value)
+          ? (builder = builder.query('terms', key, value))
+          : (builder = builder.query('term', key, value))
+      })
     const { body } = await client.search<SearchResponse<Page>>({
       index: INDEX_ALIAS,
-      body: params,
+      body: builder.build(),
     })
 
     if (body.hits.total.value === 0) {
@@ -390,6 +411,8 @@ export const searchPages = async (
       matchFilters,
       ids,
       includeContent,
+      noFilters,
+      siteName,
     } = args
     // default order is descending
     const sortOrder = sort?.order || SortOrder.DESCENDING
@@ -401,97 +424,76 @@ export const searchPages = async (
     const excludeLabels = labelFilters?.filter(
       (filter) => filter.type === LabelFilterType.EXCLUDE
     )
-
-    const body: SearchBody = {
-      query: {
-        bool: {
-          must: [
-            {
-              term: {
-                userId,
-              },
-            },
-          ],
-          should: [],
-          must_not: [],
-        },
-      },
-      sort: [
-        {
-          [sortField]: {
-            order: sortOrder,
-          },
-        },
-      ],
-      from,
-      size,
-      _source: {
+    // start building the query
+    let builder = esBuilder()
+      .query('term', { userId })
+      .sort(sortField, sortOrder)
+      .from(from)
+      .size(size)
+      .rawOption('_source', {
         excludes: includeContent ? [] : ['originalHtml', 'content'],
-      },
-    }
-
+      })
     // append filters
     if (query) {
-      appendQuery(body, query)
+      builder = appendQuery(builder, query)
     }
     if (typeFilter) {
-      appendTypeFilter(body, typeFilter)
+      builder = appendTypeFilter(builder, typeFilter)
     }
     if (inFilter !== InFilter.ALL) {
-      appendInFilter(body, inFilter)
+      builder = appendInFilter(builder, inFilter)
     }
     if (readFilter !== ReadFilter.ALL) {
-      appendReadFilter(body, readFilter)
+      builder = appendReadFilter(builder, readFilter)
     }
     if (hasFilters && hasFilters.length > 0) {
-      appendHasFilters(body, hasFilters)
+      builder = appendHasFilters(builder, hasFilters)
     }
     if (includeLabels && includeLabels.length > 0) {
-      appendIncludeLabelFilter(body, includeLabels)
+      builder = appendIncludeLabelFilter(builder, includeLabels)
     }
     if (excludeLabels && excludeLabels.length > 0) {
-      appendExcludeLabelFilter(body, excludeLabels)
+      builder = appendExcludeLabelFilter(builder, excludeLabels)
     }
     if (dateFilters && dateFilters.length > 0) {
-      appendDateFilters(body, dateFilters)
+      builder = appendDateFilters(builder, dateFilters)
     }
     if (termFilters) {
-      appendTermFilters(body, termFilters)
+      builder = appendTermFilters(builder, termFilters)
     }
     if (matchFilters) {
-      appendMatchFilters(body, matchFilters)
+      builder = appendMatchFilters(builder, matchFilters)
     }
     if (ids && ids.length > 0) {
-      appendIdsFilter(body, ids)
+      builder = appendIdsFilter(builder, ids)
     }
-
     if (args.recommendedBy) {
-      appendRecommendedBy(body, args.recommendedBy)
+      builder = appendRecommendedBy(builder, args.recommendedBy)
     }
-
     if (!args.includePending) {
-      body.query.bool.must_not.push({
-        term: {
-          state: ArticleSavingRequestStatus.Processing,
-        },
+      builder = builder.notQuery('term', {
+        state: ArticleSavingRequestStatus.Processing,
       })
     }
-
     if (!args.includeDeleted) {
-      body.query.bool.must_not.push({
-        term: {
-          state: ArticleSavingRequestStatus.Deleted,
-        },
+      builder = builder.notQuery('term', {
+        state: ArticleSavingRequestStatus.Deleted,
       })
     }
+    if (noFilters) {
+      builder = appendNoFilters(builder, noFilters)
+    }
+    if (siteName) {
+      builder = appendSiteNameFilter(builder, siteName)
+    }
+    // build the query
+    const body = builder.build()
 
-    console.log('searching pages in elastic', JSON.stringify(body))
-
-    const response = await client.search<SearchResponse<Page>, SearchBody>({
+    console.debug('searching pages in elastic', JSON.stringify(body))
+    const response = await client.search<SearchResponse<Page>, BuiltQuery>({
       index: INDEX_ALIAS,
       body,
     })
-
     if (response.body.hits.total.value === 0) {
       return [[], 0]
     }
@@ -505,6 +507,11 @@ export const searchPages = async (
       response.body.hits.total.value,
     ]
   } catch (e) {
+    if (e instanceof ResponseError) {
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+      console.error('failed to search pages in elastic', e.meta.body.error)
+      return undefined
+    }
     console.error('failed to search pages in elastic', e)
     return undefined
   }
@@ -648,8 +655,7 @@ export const searchAsYouType = async (
 
 export const updatePagesAsync = async (
   userId: string,
-  action: BulkActionType,
-  args?: PageSearchArgs
+  action: BulkActionType
 ): Promise<string | null> => {
   // default action is archive
   let must_not = [
@@ -659,7 +665,7 @@ export const updatePagesAsync = async (
       },
     },
   ]
-  let params: Record<string, any> = { archivedAt: new Date() }
+  let params: Record<string, unknown> = { archivedAt: new Date() }
   if (action === BulkActionType.Delete) {
     must_not = []
     params = { state: ArticleSavingRequestStatus.Deleted }

packages/api/src/elastic/types.ts

@@ -1,122 +1,17 @@
 // Define the type of the body for the Search request
-import { PickTuple } from '../util'
 import { PubsubClient } from '../datalayer/pubsub'
+import { PickTuple } from '../util'
 import {
   DateFilter,
   FieldFilter,
   HasFilter,
   InFilter,
   LabelFilter,
+  NoFilter,
   ReadFilter,
   SortParams,
 } from '../utils/search'
 
-export interface SearchBody {
-  query: {
-    bool: {
-      must: (
-        | {
-            term: {
-              [K: string]: string
-            }
-          }
-        | { exists: { field: string } }
-        | {
-            range: {
-              readingProgressPercent: { gte: number } | { lt: number }
-            }
-          }
-        | {
-            range: {
-              [K: string]: { gt: Date | undefined } | { lt: Date | undefined }
-            }
-          }
-        | {
-            nested: {
-              path: 'labels'
-              query: {
-                terms: {
-                  'labels.name': string[]
-                }
-              }
-            }
-          }
-        | {
-            nested: {
-              path: 'highlights'
-              query: {
-                exists: {
-                  field: 'highlights'
-                }
-              }
-            }
-          }
-        | {
-            nested: {
-              path: 'recommendations'
-              query: {
-                exists?: {
-                  field: string
-                }
-                term?: {
-                  'recommendations.name': string
-                }
-              }
-            }
-          }
-        | {
-            match: {
-              [K: string]: string
-            }
-          }
-        | {
-            terms: {
-              [K: string]: string[]
-            }
-          }
-      )[]
-      should: {
-        multi_match: {
-          query: string
-          fields: string[]
-          operator: 'and' | 'or'
-          type:
-            | 'best_fields'
-            | 'most_fields'
-            | 'cross_fields'
-            | 'phrase'
-            | 'phrase_prefix'
-        }
-      }[]
-      minimum_should_match?: number
-      must_not: (
-        | { term: { state: ArticleSavingRequestStatus } }
-        | {
-            exists: {
-              field: string
-            }
-          }
-        | {
-            nested: {
-              path: 'labels'
-              query: {
-                terms: {
-                  'labels.name': string[]
-                }
-              }[]
-            }
-          }
-      )[]
-    }
-  }
-  sort: [Record<string, { order: string }>]
-  from: number
-  size: number
-  _source: {
-    excludes: string[]
-  }
-}
-
 // Complete definition of the Search response
 export interface ShardsResponse {
   total: number
@@ -151,7 +46,7 @@ export interface SearchResponse<T> {
       _explanation?: Explanation
       fields?: never
       highlight?: never
-      inner_hits?: any
+      inner_hits?: unknown
       matched_queries?: string[]
       sort?: string[]
     }>
@@ -176,6 +71,12 @@ export enum ArticleSavingRequestStatus {
   Deleted = 'DELETED',
 }
 
+export enum HighlightType {
+  Highlight = 'HIGHLIGHT',
+  Redaction = 'REDACTION', // allowing people to remove text from the page
+  Note = 'NOTE', // allowing people to add a note at the document level
+}
+
 export interface Label {
   id: string
   name: string
@@ -187,8 +88,8 @@ export interface Label {
 export interface Highlight {
   id: string
   shortId: string
-  patch: string
-  quote: string
+  patch?: string | null
+  quote?: string | null
   userId: string
   createdAt: Date
   prefix?: string | null
@@ -199,6 +100,8 @@ export interface Highlight {
   labels?: Label[]
   highlightPositionPercent?: number | null
   highlightPositionAnchorIndex?: number | null
+  type: HighlightType
+  html?: string | null
 }
 
 export interface RecommendingUser {
@@ -231,6 +134,7 @@ export interface Page {
   originalHtml?: string | null
   slug: string
   labels?: Label[]
+  readingProgressTopPercent?: number
   readingProgressPercent: number
   readingProgressAnchorIndex: number
   createdAt: Date
@@ -272,6 +176,7 @@ export interface SearchItem {
   uploadFileId?: string | null
   url: string
   archivedAt?: Date | null
+  readingProgressTopPercent?: number
   readingProgressPercent: number
   readingProgressAnchorIndex: number
   userId: string
@@ -317,4 +222,6 @@ export interface PageSearchArgs {
   ids?: string[]
   recommendedBy?: string
   includeContent?: boolean
+  noFilters?: NoFilter[]
+  siteName?: string
 }

packages/api/src/generated/graphql.ts

@@ -110,6 +110,7 @@ export type Article = {
   readAt?: Maybe<Scalars['Date']>;
   readingProgressAnchorIndex: Scalars['Int'];
   readingProgressPercent: Scalars['Float'];
+  readingProgressTopPercent?: Maybe<Scalars['Float']>;
   recommendations?: Maybe<Array<Recommendation>>;
   savedAt: Scalars['Date'];
   savedByViewer?: Maybe<Scalars['Boolean']>;
@@ -167,6 +168,7 @@ export type ArticleSavingRequest = {
   slug: Scalars['String'];
   status: ArticleSavingRequestStatus;
   updatedAt: Scalars['Date'];
+  url: Scalars['String'];
   user: User;
   /** @deprecated userId has been replaced with user */
   userId: Scalars['ID'];
@@ -178,6 +180,7 @@ export type ArticleSavingRequestError = {
 };
 
 export enum ArticleSavingRequestErrorCode {
+  BadData = 'BAD_DATA',
   NotFound = 'NOT_FOUND',
   Unauthorized = 'UNAUTHORIZED'
 }
@@ -343,13 +346,15 @@ export type CreateHighlightInput = {
   articleId: Scalars['ID'];
   highlightPositionAnchorIndex?: InputMaybe<Scalars['Int']>;
   highlightPositionPercent?: InputMaybe<Scalars['Float']>;
+  html?: InputMaybe<Scalars['String']>;
   id: Scalars['ID'];
-  patch: Scalars['String'];
+  patch?: InputMaybe<Scalars['String']>;
   prefix?: InputMaybe<Scalars['String']>;
-  quote: Scalars['String'];
+  quote?: InputMaybe<Scalars['String']>;
   sharedAt?: InputMaybe<Scalars['Date']>;
   shortId: Scalars['String'];
   suffix?: InputMaybe<Scalars['String']>;
+  type?: InputMaybe<HighlightType>;
 };
 
 export type CreateHighlightReplyError = {
@@ -898,16 +903,18 @@ export type Highlight = {
   createdByMe: Scalars['Boolean'];
   highlightPositionAnchorIndex?: Maybe<Scalars['Int']>;
   highlightPositionPercent?: Maybe<Scalars['Float']>;
+  html?: Maybe<Scalars['String']>;
   id: Scalars['ID'];
   labels?: Maybe<Array<Label>>;
-  patch: Scalars['String'];
+  patch?: Maybe<Scalars['String']>;
   prefix?: Maybe<Scalars['String']>;
-  quote: Scalars['String'];
+  quote?: Maybe<Scalars['String']>;
   reactions: Array<Reaction>;
   replies: Array<HighlightReply>;
   sharedAt?: Maybe<Scalars['Date']>;
   shortId: Scalars['String'];
   suffix?: Maybe<Scalars['String']>;
+  type: HighlightType;
   updatedAt: Scalars['Date'];
   user: User;
 };
@@ -927,6 +934,12 @@ export type HighlightStats = {
   highlightCount: Scalars['Int'];
 };
 
+export enum HighlightType {
+  Highlight = 'HIGHLIGHT',
+  Note = 'NOTE',
+  Redaction = 'REDACTION'
+}
+
 export type Integration = {
   __typename?: 'Integration';
   createdAt: Scalars['Date'];
@@ -1119,6 +1132,7 @@ export type MergeHighlightInput = {
   articleId: Scalars['ID'];
   highlightPositionAnchorIndex?: InputMaybe<Scalars['Int']>;
   highlightPositionPercent?: InputMaybe<Scalars['Float']>;
+  html?: InputMaybe<Scalars['String']>;
   id: Scalars['ID'];
   overlapHighlightIdList: Array<Scalars['String']>;
   patch: Scalars['String'];
@@ -1747,7 +1761,8 @@ export type QueryArticleArgs = {
 
 
 export type QueryArticleSavingRequestArgs = {
-  id: Scalars['ID'];
+  id?: InputMaybe<Scalars['ID']>;
+  url?: InputMaybe<Scalars['String']>;
 };
 
 
@@ -2125,6 +2140,7 @@ export type SaveArticleReadingProgressInput = {
   id: Scalars['ID'];
   readingProgressAnchorIndex: Scalars['Int'];
   readingProgressPercent: Scalars['Float'];
+  readingProgressTopPercent?: InputMaybe<Scalars['Float']>;
 };
 
 export type SaveArticleReadingProgressResult = SaveArticleReadingProgressError | SaveArticleReadingProgressSuccess;
@@ -2233,6 +2249,7 @@ export type SearchItem = {
   readAt?: Maybe<Scalars['Date']>;
   readingProgressAnchorIndex: Scalars['Int'];
   readingProgressPercent: Scalars['Float'];
+  readingProgressTopPercent?: Maybe<Scalars['Float']>;
   recommendations?: Maybe<Array<Recommendation>>;
   savedAt: Scalars['Date'];
   shortId?: Maybe<Scalars['String']>;
@@ -2714,6 +2731,8 @@ export enum UpdateHighlightErrorCode {
 export type UpdateHighlightInput = {
   annotation?: InputMaybe<Scalars['String']>;
   highlightId: Scalars['ID'];
+  html?: InputMaybe<Scalars['String']>;
+  quote?: InputMaybe<Scalars['String']>;
   sharedAt?: InputMaybe<Scalars['Date']>;
 };
 
@@ -2813,6 +2832,7 @@ export type UpdatePageInput = {
   byline?: InputMaybe<Scalars['String']>;
   description?: InputMaybe<Scalars['String']>;
   pageId: Scalars['ID'];
+  publishedAt?: InputMaybe<Scalars['Date']>;
   savedAt?: InputMaybe<Scalars['Date']>;
   title?: InputMaybe<Scalars['String']>;
 };
@@ -3372,6 +3392,7 @@ export type ResolversTypes = {
   Highlight: ResolverTypeWrapper<Highlight>;
   HighlightReply: ResolverTypeWrapper<HighlightReply>;
   HighlightStats: ResolverTypeWrapper<HighlightStats>;
+  HighlightType: HighlightType;
   ID: ResolverTypeWrapper<Scalars['ID']>;
   Int: ResolverTypeWrapper<Scalars['Int']>;
   Integration: ResolverTypeWrapper<Integration>;
@@ -4057,6 +4078,7 @@ export type ResolversParentTypes = {
 export type SanitizeDirectiveArgs = {
   allowedTags?: Maybe<Array<Maybe<Scalars['String']>>>;
   maxLength?: Maybe<Scalars['Int']>;
+  minLength?: Maybe<Scalars['Int']>;
   pattern?: Maybe<Scalars['String']>;
 };
 
@@ -4140,6 +4162,7 @@ export type ArticleResolvers<ContextType = ResolverContext, ParentType extends R
   readAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
   readingProgressAnchorIndex?: Resolver<ResolversTypes['Int'], ParentType, ContextType>;
   readingProgressPercent?: Resolver<ResolversTypes['Float'], ParentType, ContextType>;
+  readingProgressTopPercent?: Resolver<Maybe<ResolversTypes['Float']>, ParentType, ContextType>;
   recommendations?: Resolver<Maybe<Array<ResolversTypes['Recommendation']>>, ParentType, ContextType>;
   savedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   savedByViewer?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
@@ -4183,6 +4206,7 @@ export type ArticleSavingRequestResolvers<ContextType = ResolverContext, ParentT
   slug?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   status?: Resolver<ResolversTypes['ArticleSavingRequestStatus'], ParentType, ContextType>;
   updatedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
+  url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   user?: Resolver<ResolversTypes['User'], ParentType, ContextType>;
   userId?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -4704,16 +4728,18 @@ export type HighlightResolvers<ContextType = ResolverContext, ParentType extends
   createdByMe?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
   highlightPositionAnchorIndex?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   highlightPositionPercent?: Resolver<Maybe<ResolversTypes['Float']>, ParentType, ContextType>;
+  html?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
   labels?: Resolver<Maybe<Array<ResolversTypes['Label']>>, ParentType, ContextType>;
-  patch?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  patch?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   prefix?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  quote?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  quote?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   reactions?: Resolver<Array<ResolversTypes['Reaction']>, ParentType, ContextType>;
   replies?: Resolver<Array<ResolversTypes['HighlightReply']>, ParentType, ContextType>;
   sharedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
   shortId?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   suffix?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  type?: Resolver<ResolversTypes['HighlightType'], ParentType, ContextType>;
   updatedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   user?: Resolver<ResolversTypes['User'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -5065,7 +5091,7 @@ export type ProfileResolvers<ContextType = ResolverContext, ParentType extends R
 export type QueryResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Query'] = ResolversParentTypes['Query']> = {
   apiKeys?: Resolver<ResolversTypes['ApiKeysResult'], ParentType, ContextType>;
   article?: Resolver<ResolversTypes['ArticleResult'], ParentType, ContextType, RequireFields<QueryArticleArgs, 'slug' | 'username'>>;
-  articleSavingRequest?: Resolver<ResolversTypes['ArticleSavingRequestResult'], ParentType, ContextType, RequireFields<QueryArticleSavingRequestArgs, 'id'>>;
+  articleSavingRequest?: Resolver<ResolversTypes['ArticleSavingRequestResult'], ParentType, ContextType, Partial<QueryArticleSavingRequestArgs>>;
   articles?: Resolver<ResolversTypes['ArticlesResult'], ParentType, ContextType, Partial<QueryArticlesArgs>>;
   deviceTokens?: Resolver<ResolversTypes['DeviceTokensResult'], ParentType, ContextType>;
   feedArticles?: Resolver<ResolversTypes['FeedArticlesResult'], ParentType, ContextType, Partial<QueryFeedArticlesArgs>>;
@@ -5363,6 +5389,7 @@ export type SearchItemResolvers<ContextType = ResolverContext, ParentType extend
   readAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
   readingProgressAnchorIndex?: Resolver<ResolversTypes['Int'], ParentType, ContextType>;
   readingProgressPercent?: Resolver<ResolversTypes['Float'], ParentType, ContextType>;
+  readingProgressTopPercent?: Resolver<Maybe<ResolversTypes['Float']>, ParentType, ContextType>;
   recommendations?: Resolver<Maybe<Array<ResolversTypes['Recommendation']>>, ParentType, ContextType>;
   savedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   shortId?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;

packages/api/src/generated/schema.graphql

@@ -1,4 +1,4 @@
-directive @sanitize(allowedTags: [String], maxLength: Int, pattern: String) on INPUT_FIELD_DEFINITION
+directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION
 
 type AddPopularReadError {
   errorCodes: [AddPopularReadErrorCode!]!
@@ -86,6 +86,7 @@ type Article {
   readAt: Date
   readingProgressAnchorIndex: Int!
   readingProgressPercent: Float!
+  readingProgressTopPercent: Float
   recommendations: [Recommendation!]
   savedAt: Date!
   savedByViewer: Boolean
@@ -134,6 +135,7 @@ type ArticleSavingRequest {
   slug: String!
   status: ArticleSavingRequestStatus!
   updatedAt: Date!
+  url: String!
   user: User!
   userId: ID! @deprecated(reason: "userId has been replaced with user")
 }
@@ -143,6 +145,7 @@ type ArticleSavingRequestError {
 }
 
 enum ArticleSavingRequestErrorCode {
+  BAD_DATA
   NOT_FOUND
   UNAUTHORIZED
 }
@@ -295,13 +298,15 @@ input CreateHighlightInput {
   articleId: ID!
   highlightPositionAnchorIndex: Int
   highlightPositionPercent: Float
+  html: String
   id: ID!
-  patch: String!
+  patch: String
   prefix: String
-  quote: String!
+  quote: String
   sharedAt: Date
   shortId: String!
   suffix: String
+  type: HighlightType
 }
 
 type CreateHighlightReplyError {
@@ -795,16 +800,18 @@ type Highlight {
   createdByMe: Boolean!
   highlightPositionAnchorIndex: Int
   highlightPositionPercent: Float
+  html: String
   id: ID!
   labels: [Label!]
-  patch: String!
+  patch: String
   prefix: String
-  quote: String!
+  quote: String
   reactions: [Reaction!]!
   replies: [HighlightReply!]!
   sharedAt: Date
   shortId: String!
   suffix: String
+  type: HighlightType!
   updatedAt: Date!
   user: User!
 }
@@ -822,6 +829,12 @@ type HighlightStats {
   highlightCount: Int!
 }
 
+enum HighlightType {
+  HIGHLIGHT
+  NOTE
+  REDACTION
+}
+
 type Integration {
   createdAt: Date!
   enabled: Boolean!
@@ -995,6 +1008,7 @@ input MergeHighlightInput {
   articleId: ID!
   highlightPositionAnchorIndex: Int
   highlightPositionPercent: Float
+  html: String
   id: ID!
   overlapHighlightIdList: [String!]!
   patch: String!
@@ -1241,7 +1255,7 @@ type Profile {
 type Query {
   apiKeys: ApiKeysResult!
   article(format: String, slug: String!, username: String!): ArticleResult!
-  articleSavingRequest(id: ID!): ArticleSavingRequestResult!
+  articleSavingRequest(id: ID, url: String): ArticleSavingRequestResult!
   articles(after: String, first: Int, includePending: Boolean, query: String, sharedOnly: Boolean, sort: SortParams): ArticlesResult!
   deviceTokens: DeviceTokensResult!
   feedArticles(after: String, first: Int, sharedByUser: ID, sort: SortParams): FeedArticlesResult!
@@ -1532,6 +1546,7 @@ input SaveArticleReadingProgressInput {
   id: ID!
   readingProgressAnchorIndex: Int!
   readingProgressPercent: Float!
+  readingProgressTopPercent: Float
 }
 
 union SaveArticleReadingProgressResult = SaveArticleReadingProgressError | SaveArticleReadingProgressSuccess
@@ -1633,6 +1648,7 @@ type SearchItem {
   readAt: Date
   readingProgressAnchorIndex: Int!
   readingProgressPercent: Float!
+  readingProgressTopPercent: Float
   recommendations: [Recommendation!]
   savedAt: Date!
   shortId: String
@@ -2075,6 +2091,8 @@ enum UpdateHighlightErrorCode {
 input UpdateHighlightInput {
   annotation: String
   highlightId: ID!
+  html: String
+  quote: String
   sharedAt: Date
 }
 
@@ -2166,6 +2184,7 @@ input UpdatePageInput {
   byline: String
   description: String
   pageId: ID!
+  publishedAt: Date
   savedAt: Date
   title: String
 }

packages/api/src/graphql.d.ts

@@ -8,7 +8,7 @@ declare module '*.graphql' {
 }
 
 declare module 'knex-stringcase' {
-  import * as Knex from 'knex'
+  import { Knex } from 'knex'
 
   type StringCase =
     | 'camelcase'

packages/api/src/resolvers/article/index.ts

@@ -3,6 +3,26 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-floating-promises */
+import { Readability } from '@omnivore/readability'
+import * as httpContext from 'express-http-context'
+import graphqlFields from 'graphql-fields'
+import normalizeUrl from 'normalize-url'
+import { searchHighlights } from '../../elastic/highlights'
+import {
+  createPage,
+  getPageByParam,
+  searchAsYouType,
+  searchPages,
+  updatePage,
+  updatePagesAsync,
+} from '../../elastic/pages'
+import {
+  ArticleSavingRequestStatus,
+  Page,
+  PageType,
+  SearchItem as SearchItemData,
+} from '../../elastic/types'
+import { env } from '../../env'
 import {
   Article,
   ArticleError,
@@ -51,11 +71,13 @@ import {
   UpdatesSinceErrorCode,
   UpdatesSinceSuccess,
 } from '../../generated/graphql'
+import { createPageSaveRequest } from '../../services/create_page_save_request'
+import { parsedContentToPage } from '../../services/save_page'
+import { saveSearchHistory } from '../../services/search_history'
+import { traceAs } from '../../tracing'
 import { Merge } from '../../util'
-import {
-  getStorageFileDetails,
-  makeStorageFilePublic,
-} from '../../utils/uploads'
+import { analytics } from '../../utils/analytics'
+import { isSiteBlockedForParse } from '../../utils/blocked'
 import { ContentParseError } from '../../utils/errors'
 import {
   authorized,
@@ -67,45 +89,19 @@ import {
   userDataToUser,
   validatedDate,
 } from '../../utils/helpers'
+import { createImageProxyUrl } from '../../utils/imageproxy'
 import {
   getDistillerResult,
   htmlToMarkdown,
   ParsedContentPuppeteer,
   parsePreparedContent,
 } from '../../utils/parser'
-import { isSiteBlockedForParse } from '../../utils/blocked'
-import { Readability } from '@omnivore/readability'
-import { traceAs } from '../../tracing'
-
-import { createImageProxyUrl } from '../../utils/imageproxy'
-import normalizeUrl from 'normalize-url'
-import { WithDataSourcesContext } from '../types'
-
 import { parseSearchQuery, SortBy, SortOrder } from '../../utils/search'
-import { createPageSaveRequest } from '../../services/create_page_save_request'
-import { analytics } from '../../utils/analytics'
-import { env } from '../../env'
-import graphqlFields from 'graphql-fields'
-
 import {
-  ArticleSavingRequestStatus,
-  Page,
-  PageType,
-  SearchItem as SearchItemData,
-} from '../../elastic/types'
-import {
-  createPage,
-  getPageById,
-  getPageByParam,
-  searchAsYouType,
-  searchPages,
-  updatePage,
-  updatePagesAsync,
-} from '../../elastic/pages'
-import { searchHighlights } from '../../elastic/highlights'
-import { saveSearchHistory } from '../../services/search_history'
-import { parsedContentToPage } from '../../services/save_page'
-import * as httpContext from 'express-http-context'
+  getStorageFileDetails,
+  makeStorageFilePublic,
+} from '../../utils/uploads'
+import { WithDataSourcesContext } from '../types'
 
 enum ArticleFormat {
   Markdown = 'markdown',
@@ -649,24 +645,18 @@ export const setBookmarkArticleResolver = authorized<
     { input: { articleID, bookmark } },
     { claims: { uid }, log, pubsub }
   ) => {
-    const page = await getPageById(articleID)
+    const page = await getPageByParam({
+      userId: uid,
+      _id: articleID,
+    })
     if (!page) {
       return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] }
     }
 
     if (!bookmark) {
-      const pageRemoved = await getPageByParam({
-        userId: uid,
-        _id: articleID,
-      })
-
-      if (!pageRemoved) {
-        return { errorCodes: [SetBookmarkArticleErrorCode.NotFound] }
-      }
-
       // delete the page and its metadata
       const deleted = await updatePage(
-        pageRemoved.id,
+        page.id,
         {
           state: ArticleSavingRequestStatus.Deleted,
           labels: [],
@@ -684,7 +674,7 @@ export const setBookmarkArticleResolver = authorized<
         userId: uid,
         event: 'link_removed',
         properties: {
-          url: pageRemoved.url,
+          url: page.url,
           env: env.server.apiEnv,
         },
       })
@@ -704,7 +694,7 @@ export const setBookmarkArticleResolver = authorized<
       // Make sure article.id instead of userArticle.id has passed. We use it for cache updates
       return {
         bookmarkedArticle: {
-          ...pageRemoved,
+          ...page,
           isArchived: false,
           savedByViewer: false,
           postedByViewer: false,
@@ -764,7 +754,14 @@ export const saveArticleReadingProgressResolver = authorized<
 >(
   async (
     _,
-    { input: { id, readingProgressPercent, readingProgressAnchorIndex } },
+    {
+      input: {
+        id,
+        readingProgressPercent,
+        readingProgressAnchorIndex,
+        readingProgressTopPercent,
+      },
+    },
     { claims: { uid }, pubsub }
   ) => {
     const page = await getPageByParam({ userId: uid, _id: id })
@@ -774,31 +771,43 @@ export const saveArticleReadingProgressResolver = authorized<
     }
 
     if (
-      (!readingProgressPercent && readingProgressPercent !== 0) ||
       readingProgressPercent < 0 ||
-      readingProgressPercent > 100
+      readingProgressPercent > 100 ||
+      (readingProgressTopPercent &&
+        (readingProgressTopPercent < 0 ||
+          readingProgressTopPercent > readingProgressPercent)) ||
+      readingProgressAnchorIndex < 0
     ) {
       return { errorCodes: [SaveArticleReadingProgressErrorCode.BadData] }
     }
-
+    // If we have a top percent, we only save it if it's greater than the current top percent
+    // or set to zero if the top percent is zero.
+    const readingProgressTopPercentToSave = readingProgressTopPercent
+      ? Math.max(readingProgressTopPercent, page.readingProgressTopPercent || 0)
+      : readingProgressTopPercent === 0
+      ? 0
+      : undefined
     // If setting to zero we accept the update, otherwise we require it
     // be greater than the current reading progress.
-    const shouldUpdate =
-      readingProgressPercent === 0 ||
-      page.readingProgressPercent < readingProgressPercent ||
-      page.readingProgressAnchorIndex < readingProgressAnchorIndex
-
     const updatedPart = {
-      readingProgressPercent: shouldUpdate
-        ? readingProgressPercent
-        : page.readingProgressPercent,
-      readingProgressAnchorIndex: shouldUpdate
-        ? readingProgressAnchorIndex
-        : page.readingProgressAnchorIndex,
+      readingProgressPercent:
+        readingProgressPercent === 0
+          ? 0
+          : Math.max(readingProgressPercent, page.readingProgressPercent),
+      readingProgressAnchorIndex:
+        readingProgressAnchorIndex === 0
+          ? 0
+          : Math.max(
+              readingProgressAnchorIndex,
+              page.readingProgressAnchorIndex
+            ),
+      readingProgressTopPercent: readingProgressTopPercentToSave,
       readAt: new Date(),
     }
-
-    await updatePage(id, updatedPart, { pubsub, uid })
+    const updated = await updatePage(id, updatedPart, { pubsub, uid })
+    if (!updated) {
+      return { errorCodes: [SaveArticleReadingProgressErrorCode.NotFound] }
+    }
 
     return {
       updatedArticle: {

packages/api/src/resolvers/article_saving_request/index.ts

@@ -1,4 +1,6 @@
 /* eslint-disable prefer-const */
+import { getPageByParam } from '../../elastic/pages'
+import { env } from '../../env'
 import {
   ArticleSavingRequestError,
   ArticleSavingRequestErrorCode,
@@ -10,16 +12,14 @@ import {
   MutationCreateArticleSavingRequestArgs,
   QueryArticleSavingRequestArgs,
 } from '../../generated/graphql'
+import { createPageSaveRequest } from '../../services/create_page_save_request'
+import { analytics } from '../../utils/analytics'
 import {
   authorized,
   isParsingTimeout,
   pageToArticleSavingRequest,
 } from '../../utils/helpers'
-import { createPageSaveRequest } from '../../services/create_page_save_request'
-import { getPageById } from '../../elastic/pages'
 import { isErrorWithCode } from '../user'
-import { analytics } from '../../utils/analytics'
-import { env } from '../../env'
 
 export const createArticleSavingRequestResolver = authorized<
   CreateArticleSavingRequestSuccess,
@@ -56,23 +56,29 @@ export const articleSavingRequestResolver = authorized<
   ArticleSavingRequestSuccess,
   ArticleSavingRequestError,
   QueryArticleSavingRequestArgs
->(async (_, { id }, { models }) => {
-  let page
-  let user
-  try {
-    page = await getPageById(id)
-    if (!page) {
-      return { errorCodes: [ArticleSavingRequestErrorCode.NotFound] }
-    }
-    user = await models.user.get(page.userId)
-    // eslint-disable-next-line no-empty
-  } catch (error) {}
-  if (user && page) {
-    if (isParsingTimeout(page)) {
-      page.state = ArticleSavingRequestStatus.Succeeded
-    }
-    return { articleSavingRequest: pageToArticleSavingRequest(user, page) }
+>(async (_, { id, url }, { models, claims }) => {
+  if (!id && !url) {
+    return { errorCodes: [ArticleSavingRequestErrorCode.BadData] }
   }
-
-  return { errorCodes: [ArticleSavingRequestErrorCode.NotFound] }
+  const user = await models.user.get(claims.uid)
+  if (!user) {
+    return { errorCodes: [ArticleSavingRequestErrorCode.Unauthorized] }
+  }
+  const params = {
+    _id: id || undefined,
+    url: url || undefined,
+    userId: claims.uid,
+    state: [
+      ArticleSavingRequestStatus.Succeeded,
+      ArticleSavingRequestStatus.Processing,
+    ],
+  }
+  const page = await getPageByParam(params)
+  if (!page) {
+    return { errorCodes: [ArticleSavingRequestErrorCode.NotFound] }
+  }
+  if (isParsingTimeout(page)) {
+    page.state = ArticleSavingRequestStatus.Succeeded
+  }
+  return { articleSavingRequest: pageToArticleSavingRequest(user, page) }
 })

packages/api/src/resolvers/function_resolvers.ts

@@ -3,23 +3,28 @@
 /* eslint-disable @typescript-eslint/no-unsafe-return */
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types */
-import { createReactionResolver, deleteReactionResolver } from './reaction'
-import { Claims, WithDataSourcesContext } from './types'
-import { createImageProxyUrl } from '../utils/imageproxy'
-import { userDataToUser, validatedDate, wordsCount } from '../utils/helpers'
-
+import { getShareInfoForArticle } from '../datalayer/links/share_info'
+import { getPageByParam } from '../elastic/pages'
 import {
   Article,
   ArticleHighlightsInput,
   ContentReader,
   Highlight,
+  HighlightType,
   LinkShareInfo,
   PageType,
   Reaction,
   SearchItem,
   User,
 } from '../generated/graphql'
-
+import { userDataToUser, validatedDate, wordsCount } from '../utils/helpers'
+import { createImageProxyUrl } from '../utils/imageproxy'
+import {
+  generateDownloadSignedUrl,
+  generateUploadFilePathName,
+} from '../utils/uploads'
+import { optInFeatureResolver } from './features'
+import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
 import {
   addPopularReadResolver,
   apiKeysResolver,
@@ -109,16 +114,10 @@ import {
   webhookResolver,
   webhooksResolver,
 } from './index'
-import { getShareInfoForArticle } from '../datalayer/links/share_info'
-import {
-  generateDownloadSignedUrl,
-  generateUploadFilePathName,
-} from '../utils/uploads'
-import { getPageByParam } from '../elastic/pages'
-import { recentSearchesResolver } from './recent_searches'
-import { optInFeatureResolver } from './features'
-import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
+import { createReactionResolver, deleteReactionResolver } from './reaction'
 import { markEmailAsItemResolver, recentEmailsResolver } from './recent_emails'
+import { recentSearchesResolver } from './recent_searches'
+import { Claims, WithDataSourcesContext } from './types'
 
 /* eslint-disable @typescript-eslint/naming-convention */
 type ResultResolveType = {
@@ -471,33 +470,11 @@ export const functionResolvers = {
         ? ContentReader.Pdf
         : ContentReader.Web
     },
-    async highlights(
+    highlights(
       article: { id: string; userId?: string; highlights?: Highlight[] },
       _: { input: ArticleHighlightsInput },
       ctx: WithDataSourcesContext
     ) {
-      // const includeFriends = false
-      // // TODO: this is a temporary solution until we figure out how collaborative approach would look like
-      // // article has userId only if it's returned by getSharedArticle resolver
-      // if (article.userId) {
-      //   const result = await ctx.models.highlight.getForUserArticle(
-      //     article.userId,
-      //     article.id
-      //   )
-      //   return result
-      // }
-      //
-      // const friendsIds =
-      //   ctx.claims?.uid && includeFriends
-      //     ? await ctx.models.userFriends.getFriends(ctx.claims?.uid)
-      //     : []
-      //
-      // // FIXME: Move this filtering logic to the datalayer
-      // return (await ctx.models.highlight.batchGet(article.id)).filter((h) =>
-      //   [...(includeFriends ? friendsIds : []), ctx.claims?.uid || ''].some(
-      //     (u) => u === h.userId
-      //   )
-      // )
       return article.highlights || []
     },
     async shareInfo(
@@ -557,6 +534,9 @@ export const functionResolvers = {
     ) {
       return highlight.createdByMe ?? highlight.userId === ctx.claims?.uid
     },
+    type(highlight: { type: HighlightType }) {
+      return highlight.type || HighlightType.Highlight
+    },
   },
   Reaction: {
     async user(

packages/api/src/resolvers/highlight/index.ts

@@ -1,7 +1,19 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 /* eslint-disable @typescript-eslint/require-await */
 /* eslint-disable @typescript-eslint/no-floating-promises */
-import { authorized, unescapeHtml } from '../../utils/helpers'
+import {
+  addHighlightToPage,
+  deleteHighlight,
+  getHighlightById,
+  updateHighlight,
+} from '../../elastic/highlights'
+import { getPageById, updatePage } from '../../elastic/pages'
+import {
+  Highlight as HighlightData,
+  HighlightType,
+  Label,
+} from '../../elastic/types'
+import { env } from '../../env'
 import {
   CreateHighlightError,
   CreateHighlightErrorCode,
@@ -26,16 +38,8 @@ import {
   UpdateHighlightSuccess,
   User,
 } from '../../generated/graphql'
-import { env } from '../../env'
 import { analytics } from '../../utils/analytics'
-import { Highlight as HighlightData } from '../../elastic/types'
-import { getPageById, updatePage } from '../../elastic/pages'
-import {
-  addHighlightToPage,
-  deleteHighlight,
-  getHighlightById,
-  updateHighlight,
-} from '../../elastic/highlights'
+import { authorized, unescapeHtml } from '../../utils/helpers'
 
 const highlightDataToHighlight = (highlight: HighlightData): Highlight => ({
   ...highlight,
@@ -58,16 +62,11 @@ export const createHighlightResolver = authorized<
       errorCodes: [CreateHighlightErrorCode.NotFound],
     }
   }
-
-  analytics.track({
-    userId: claims.uid,
-    event: 'highlight_created',
-    properties: {
-      pageId,
-      env: env.server.apiEnv,
-    },
-  })
-
+  if (page.userId !== claims.uid) {
+    return {
+      errorCodes: [CreateHighlightErrorCode.Unauthorized],
+    }
+  }
   if (input.annotation && input.annotation.length > 4000) {
     return {
       errorCodes: [CreateHighlightErrorCode.BadData],
@@ -86,6 +85,7 @@ export const createHighlightResolver = authorized<
       createdAt: new Date(),
       userId: claims.uid,
       annotation,
+      type: input.type || HighlightType.Highlight,
     }
 
     if (
@@ -108,6 +108,15 @@ export const createHighlightResolver = authorized<
       },
     })
 
+    analytics.track({
+      userId: claims.uid,
+      event: 'highlight_created',
+      properties: {
+        pageId,
+        env: env.server.apiEnv,
+      },
+    })
+
     return { highlight: highlightDataToHighlight(highlight) }
   } catch (err) {
     log.error('Error creating highlight', err)
@@ -130,39 +139,53 @@ export const mergeHighlightResolver = authorized<
       errorCodes: [MergeHighlightErrorCode.NotFound],
     }
   }
-
-  const articleHighlights = page.highlights
-
+  if (page.userId !== claims.uid) {
+    return {
+      errorCodes: [MergeHighlightErrorCode.Unauthorized],
+    }
+  }
   /* Compute merged annotation form the order of highlights appearing on page */
-  const overlapAnnotations: { [id: string]: string } = {}
-  articleHighlights.forEach((highlight, index) => {
-    if (overlapHighlightIdList.includes(highlight.id)) {
-      articleHighlights.splice(index, 1)
-
+  const mergedAnnotations: string[] = []
+  const mergedLabels: Label[] = []
+  const pageHighlights = page.highlights.filter((highlight) => {
+    // filter out highlights that are in the overlap list
+    // and are of type highlight (not annotation or note)
+    if (
+      overlapHighlightIdList.includes(highlight.id) &&
+      highlight.type === HighlightType.Highlight
+    ) {
       if (highlight.annotation) {
-        overlapAnnotations[highlight.id] = highlight.annotation
+        mergedAnnotations.push(highlight.annotation)
       }
+      if (highlight.labels) {
+        // remove duplicates from labels by checking id
+        highlight.labels.forEach((label) => {
+          if (
+            !mergedLabels.find((mergedLabel) => mergedLabel.id === label.id)
+          ) {
+            mergedLabels.push(label)
+          }
+        })
+      }
+      return false
     }
+    return true
   })
-  const mergedAnnotation: string[] = []
-  overlapHighlightIdList.forEach((highlightId) => {
-    if (overlapAnnotations[highlightId]) {
-      mergedAnnotation.push(overlapAnnotations[highlightId])
-    }
-  })
-
   try {
     const highlight: HighlightData = {
       ...newHighlightInput,
       updatedAt: new Date(),
       createdAt: new Date(),
       userId: claims.uid,
-      annotation: mergedAnnotation ? mergedAnnotation.join('\n') : null,
+      annotation:
+        mergedAnnotations.length > 0 ? mergedAnnotations.join('\n') : null,
+      type: HighlightType.Highlight,
+      labels: mergedLabels,
     }
 
     const merged = await updatePage(
       pageId,
-      { highlights: articleHighlights.concat(highlight) },
+      { highlights: pageHighlights.concat(highlight) },
       { pubsub, uid: claims.uid }
     )
     if (!merged) {
@@ -204,8 +227,7 @@ export const updateHighlightResolver = authorized<
   UpdateHighlightError,
   MutationUpdateHighlightArgs
 >(async (_, { input }, { pubsub, claims, log }) => {
-  const { highlightId } = input
-  const highlight = await getHighlightById(highlightId)
+  const highlight = await getHighlightById(input.highlightId)
 
   if (!highlight?.id) {
     return {
@@ -219,20 +241,16 @@ export const updateHighlightResolver = authorized<
     }
   }
 
-  if (input.annotation && input.annotation.length > 4000) {
-    return {
-      errorCodes: [UpdateHighlightErrorCode.BadData],
-    }
-  }
-
   // unescape HTML entities
   const annotation = input.annotation
     ? unescapeHtml(input.annotation)
     : undefined
+  const quote = input.quote ? unescapeHtml(input.quote) : highlight.quote
 
   const updatedHighlight: HighlightData = {
     ...highlight,
     annotation,
+    quote,
     updatedAt: new Date(),
   }
 

packages/api/src/resolvers/labels/index.ts

@@ -243,12 +243,20 @@ export const setLabelsResolver = authorized<
         errorCodes: [SetLabelsErrorCode.NotFound],
       }
     }
-
+    // filter out labels that are already set
+    const labelsToAdd = labels.filter(
+      (label) => !page.labels?.some((pageLabel) => pageLabel.id === label.id)
+    )
     // update labels in the page
-    const updated = await updateLabelsInPage(pageId, labels, {
-      pubsub,
-      uid,
-    })
+    const updated = await updateLabelsInPage(
+      pageId,
+      labels,
+      {
+        pubsub,
+        uid,
+      },
+      labelsToAdd
+    )
     if (!updated) {
       return {
         errorCodes: [SetLabelsErrorCode.NotFound],

packages/api/src/resolvers/reminders/index.ts

@@ -1,4 +1,7 @@
-import { authorized } from '../../utils/helpers'
+import { DateTime } from 'luxon'
+import { getPageById } from '../../elastic/pages'
+import { Page } from '../../elastic/types'
+import { env } from '../../env'
 import {
   CreateReminderError,
   CreateReminderErrorCode,
@@ -17,14 +20,11 @@ import {
   UpdateReminderErrorCode,
   UpdateReminderSuccess,
 } from '../../generated/graphql'
-import { deleteTask, enqueueReminder } from '../../utils/createTask'
-import { analytics } from '../../utils/analytics'
-import { env } from '../../env'
-import { DataModels } from '../types'
-import { DateTime } from 'luxon'
 import { setLinkArchived } from '../../services/archive_link'
-import { getPageById } from '../../elastic/pages'
-import { Page } from '../../elastic/types'
+import { analytics } from '../../utils/analytics'
+import { deleteTask, enqueueReminder } from '../../utils/createTask'
+import { authorized } from '../../utils/helpers'
+import { DataModels } from '../types'
 
 const validScheduleTime = (str: string): Date | undefined => {
   const scheduleTime = DateTime.fromISO(str, { setZone: true }).set({
@@ -166,7 +166,11 @@ export const reminderResolver = authorized<
         errorCodes: [ReminderErrorCode.NotFound],
       }
     }
-
+    if (page.userId !== uid) {
+      return {
+        errorCodes: [ReminderErrorCode.Unauthorized],
+      }
+    }
     const reminder = await models.reminder.getCreatedByParameters(uid, {
       elasticPageId: page.id,
     })

packages/api/src/resolvers/types.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/ban-types */
 import { Context as ApolloContext } from 'apollo-server-core'
 import winston from 'winston'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import UserModel from '../datalayer/user'
 import ArticleModel from '../datalayer/article'
 import UserArticleModel from '../datalayer/links'

packages/api/src/resolvers/update/index.ts

@@ -41,6 +41,7 @@ export const updatePageResolver = authorized<
     description: input.description ?? undefined,
     author: input.byline ?? undefined,
     savedAt: input.savedAt ? new Date(input.savedAt) : undefined,
+    publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined,
   }
 
   const updateResult = await updatePage(input.pageId, pageData, { ...ctx, uid })

packages/api/src/resolvers/user_friends/index.ts

@@ -1,18 +1,18 @@
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { UserData } from '../../datalayer/user/model'
 import {
-  User,
-  ResolverFn,
-  SetFollowSuccess,
-  SetFollowError,
-  MutationSetFollowArgs,
-  SetFollowErrorCode,
   GetFollowersResult,
   GetFollowingResult,
+  MutationSetFollowArgs,
   QueryGetFollowersArgs,
   QueryGetFollowingArgs,
+  ResolverFn,
+  SetFollowError,
+  SetFollowErrorCode,
+  SetFollowSuccess,
+  User,
 } from '../../generated/graphql'
-import { userDataToUser, authorized } from '../../utils/helpers'
+import { authorized, userDataToUser } from '../../utils/helpers'
 import { DataModels, WithDataSourcesContext } from '../types'
 
 export const setFollowResolver = authorized<

packages/api/src/resolvers/webhooks/index.ts

@@ -194,20 +194,7 @@ export const setWebhookResolver = authorized<
       }
 
       webhookToSave.id = input.id
-    } else {
-      // Create
-      const existingWebhook = await getRepository(Webhook).findOneBy({
-        user: { id: uid },
-        eventTypes: `{${input.eventTypes.join(',')}}`,
-      })
-
-      if (existingWebhook) {
-        return {
-          errorCodes: [SetWebhookErrorCode.AlreadyExists],
-        }
-      }
     }
-
     const webhook = await getRepository(Webhook).save({
       user,
       ...webhookToSave,

packages/api/src/routers/article_router.ts

@@ -1,25 +1,25 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import express from 'express'
-import { CreateArticleErrorCode } from '../generated/graphql'
-import { isSiteBlockedForParse } from '../utils/blocked'
-import cors from 'cors'
-import { buildLogger } from '../utils/logger'
-import { corsConfig } from '../utils/corsConfig'
-import { createPageSaveRequest } from '../services/create_page_save_request'
-import { initModels } from '../server'
-import { kx } from '../datalayer/knex_config'
-import { getClaimsByToken } from '../utils/auth'
-import * as jwt from 'jsonwebtoken'
-import { env } from '../env'
-import { Claims } from '../resolvers/types'
-import { getRepository } from '../entity/utils'
-import { Speech, SpeechState } from '../entity/speech'
-import { getPageById, updatePage } from '../elastic/pages'
-import { generateDownloadSignedUrl } from '../utils/uploads'
-import { enqueueTextToSpeech } from '../utils/createTask'
-import { createPubSubClient } from '../datalayer/pubsub'
 import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler'
+import cors from 'cors'
+import express from 'express'
+import * as jwt from 'jsonwebtoken'
+import { kx } from '../datalayer/knex_config'
+import { createPubSubClient } from '../datalayer/pubsub'
+import { getPageById, updatePage } from '../elastic/pages'
+import { Speech, SpeechState } from '../entity/speech'
+import { getRepository } from '../entity/utils'
+import { env } from '../env'
+import { CreateArticleErrorCode } from '../generated/graphql'
+import { Claims } from '../resolvers/types'
+import { initModels } from '../server'
+import { createPageSaveRequest } from '../services/create_page_save_request'
+import { getClaimsByToken } from '../utils/auth'
+import { isSiteBlockedForParse } from '../utils/blocked'
+import { corsConfig } from '../utils/corsConfig'
+import { enqueueTextToSpeech } from '../utils/createTask'
+import { buildLogger } from '../utils/logger'
+import { generateDownloadSignedUrl } from '../utils/uploads'
 
 interface SpeechInput {
   voice?: string
@@ -74,6 +74,7 @@ export function articleRouter() {
 
     return res.send({
       articleSavingRequestId: result.id,
+      url: result.url,
     })
   })
 
@@ -110,6 +111,13 @@ export function articleRouter() {
           if (!page) {
             return res.status(404).send('Page not found')
           }
+          if (page.userId !== uid) {
+            logger.info('User is not allowed to access speech of the article', {
+              userId: uid,
+              articleId,
+            })
+            return res.status(401).send({ errorCode: 'UNAUTHORIZED' })
+          }
           const speechFile = htmlToSpeechFile({
             title: page.title,
             content: page.content,

packages/api/src/routers/auth/auth_router.ts

@@ -42,9 +42,9 @@ import {
   hashPassword,
   setAuthInCookie,
 } from '../../utils/auth'
-import { createUser } from '../../services/create_user'
+import { createUser, getUserByEmail } from '../../services/create_user'
 import { isErrorWithCode } from '../../resolvers'
-import { AppDataSource, initModels } from '../../server'
+import { AppDataSource } from '../../server'
 import { getRepository, setClaims } from '../../entity/utils'
 import { User } from '../../entity/user'
 import {
@@ -373,19 +373,26 @@ export function authRouter() {
     '/email-login',
     cors<express.Request>(corsConfig),
     async (req: express.Request, res: express.Response) => {
-      const { email, password } = req.body
-
-      if (!email || !password) {
+      interface LoginRequest {
+        email: string
+        password: string
+      }
+      function isValidLoginRequest(obj: any): obj is LoginRequest {
+        return (
+          'email' in obj &&
+          obj.email.trim().length > 0 && // email must not be empty
+          'password' in obj &&
+          obj.password.length >= 8 // password must be at least 8 characters
+        )
+      }
+      if (!isValidLoginRequest(req.body)) {
         return res.redirect(
           `${env.client.url}/auth/email-login?errorCodes=${LoginErrorCode.InvalidCredentials}`
         )
       }
-
+      const { email, password } = req.body
       try {
-        const models = initModels(kx, false)
-        const user = await models.user.getWhere({
-          email,
-        })
+        const user = await getUserByEmail(email.trim())
         if (!user?.id) {
           return res.redirect(
             `${env.client.url}/auth/email-login?errorCodes=${LoginErrorCode.UserNotFound}`
@@ -409,7 +416,6 @@ export function authRouter() {
             `${env.client.url}/auth/email-login?errorCodes=${LoginErrorCode.WrongSource}`
           )
         }
-
         // check if password is correct
         const validPassword = await comparePassword(password, user.password)
         if (!validPassword) {
@@ -437,25 +443,43 @@ export function authRouter() {
     '/email-signup',
     cors<express.Request>(corsConfig),
     async (req: express.Request, res: express.Response) => {
-      const { email, password, name, username, bio, pictureUrl } = req.body
-
-      if (!email || !password || !name || !username) {
+      interface SignupRequest {
+        email: string
+        password: string
+        name: string
+        username: string
+        bio?: string
+        pictureUrl?: string
+      }
+      function isValidSignupRequest(obj: any): obj is SignupRequest {
+        return (
+          'email' in obj &&
+          obj.email.trim().length > 0 && // email must not be empty
+          'password' in obj &&
+          obj.password.length >= 8 && // password must be at least 8 characters
+          'name' in obj &&
+          obj.name.trim().length > 0 && // name must not be empty
+          'username' in obj &&
+          obj.username.trim().length > 0 // username must not be empty
+        )
+      }
+      if (!isValidSignupRequest(req.body)) {
         return res.redirect(
           `${env.client.url}/auth/email-signup?errorCodes=INVALID_CREDENTIALS`
         )
       }
-      const lowerCasedUsername = username.toLowerCase()
-
+      const { email, password, name, username, bio, pictureUrl } = req.body
+      // trim whitespace in email address
+      const trimmedEmail = email.trim()
       try {
         // hash password
         const hashedPassword = await hashPassword(password)
-
         await createUser({
-          email,
+          email: trimmedEmail,
           provider: 'EMAIL',
-          sourceUserId: email,
-          name,
-          username: lowerCasedUsername,
+          sourceUserId: trimmedEmail,
+          name: name.trim(),
+          username: username.trim().toLowerCase(), // lowercase username
           pictureUrl,
           bio,
           password: hashedPassword,
@@ -547,7 +571,7 @@ export function authRouter() {
     '/forgot-password',
     cors<express.Request>(corsConfig),
     async (req: express.Request, res: express.Response) => {
-      const email = req.body.email
+      const email = req.body.email?.trim() as string // trim whitespace
       if (!email) {
         return res.redirect(
           `${env.client.url}/auth/forgot-password?errorCodes=INVALID_EMAIL`
@@ -555,9 +579,7 @@ export function authRouter() {
       }
 
       try {
-        const user = await getRepository(User).findOneBy({
-          email,
-        })
+        const user = await getUserByEmail(email)
         if (!user) {
           return res.redirect(`${env.client.url}/auth/reset-sent`)
         }

packages/api/src/routers/svc/integrations.ts

@@ -3,12 +3,12 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 import express from 'express'
 import { EntityType, readPushSubscription } from '../../datalayer/pubsub'
-import { getRepository } from '../../entity/utils'
-import { Integration, IntegrationType } from '../../entity/integration'
-import { buildLogger } from '../../utils/logger'
-import { syncWithIntegration } from '../../services/integrations'
 import { getPageById, searchPages } from '../../elastic/pages'
 import { Page } from '../../elastic/types'
+import { Integration, IntegrationType } from '../../entity/integration'
+import { getRepository } from '../../entity/utils'
+import { syncWithIntegration } from '../../services/integrations'
+import { buildLogger } from '../../utils/logger'
 import { DateFilter } from '../../utils/search'
 
 export interface Message {
@@ -89,6 +89,10 @@ export function integrationsServiceRouter() {
           res.status(200).send('No page found')
           return
         }
+        if (page.userId !== userId) {
+          logger.info('Page does not belong to user', { id, userId })
+          return res.status(200).send('Page does not belong to user')
+        }
         // sync updated page with integration
         logger.info('syncing updated page with integration', {
           integrationId: integration.id,

packages/api/src/routers/text_to_speech.ts

@@ -1,22 +1,22 @@
 /* eslint-disable @typescript-eslint/no-misused-promises */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types */
-import express from 'express'
-import cors from 'cors'
-import { corsConfig } from '../utils/corsConfig'
-import { getRepository, setClaims } from '../entity/utils'
-import { getPageById } from '../elastic/pages'
-import { Speech, SpeechState } from '../entity/speech'
-import { buildLogger } from '../utils/logger'
-import { getClaimsByToken } from '../utils/auth'
-import { shouldSynthesize } from '../services/speech'
-import { readPushSubscription } from '../datalayer/pubsub'
-import { AppDataSource } from '../server'
-import { enqueueTextToSpeech } from '../utils/createTask'
 import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler'
-import { UserPersonalization } from '../entity/user_personalization'
+import cors from 'cors'
+import express from 'express'
+import { readPushSubscription } from '../datalayer/pubsub'
+import { getPageById } from '../elastic/pages'
 import { ArticleSavingRequestStatus } from '../elastic/types'
+import { Speech, SpeechState } from '../entity/speech'
+import { UserPersonalization } from '../entity/user_personalization'
+import { getRepository, setClaims } from '../entity/utils'
+import { AppDataSource } from '../server'
 import { FeatureName, getFeature } from '../services/features'
+import { shouldSynthesize } from '../services/speech'
+import { getClaimsByToken } from '../utils/auth'
+import { corsConfig } from '../utils/corsConfig'
+import { enqueueTextToSpeech } from '../utils/createTask'
+import { buildLogger } from '../utils/logger'
 
 const DEFAULT_VOICE = 'Larry'
 const DEFAULT_COMPLIMENTARY_VOICE = 'Evelyn'
@@ -59,7 +59,10 @@ export function textToSpeechRouter() {
         logger.info('No page found', { id })
         return res.status(200).send('No page found')
       }
-
+      if (page.userId !== userId) {
+        logger.info('Page does not belong to user', { id, userId })
+        return res.status(200).send('Page does not belong to user')
+      }
       if (page.state === ArticleSavingRequestStatus.Processing) {
         logger.info('Page is still processing, try again later', { id })
         return res.status(400).send('Page is still processing')

packages/api/src/scalars.ts

@@ -9,6 +9,7 @@ export class SanitizedString extends GraphQLScalarType {
     type: GraphQLScalarType,
     allowedTags?: string[],
     maxLength?: number,
+    minLength?: number,
     pattern?: string
   ) {
     super({
@@ -25,11 +26,7 @@ export class SanitizedString extends GraphQLScalarType {
 
       // invoked when a query is passed as a JSON object (for example, when Apollo Client makes a request
       parseValue(value) {
-        if (maxLength && maxLength < value.length) {
-          throw new Error(
-            `Specified value cannot be longer than ${maxLength} characters`
-          )
-        }
+        checkLength(value)
         if (pattern && !new RegExp(pattern).test(value)) {
           throw new Error(`Specified value does not match pattern`)
         }
@@ -39,17 +36,26 @@ export class SanitizedString extends GraphQLScalarType {
       // invoked when a query is passed as a string
       parseLiteral(ast) {
         const value = type.parseLiteral(ast, {})
-        if (maxLength && maxLength < value.length) {
-          throw new Error(
-            `Specified value cannot be longer than ${maxLength} characters`
-          )
-        }
+        checkLength(value)
         if (pattern && !new RegExp(pattern).test(value)) {
           throw new Error(`Specified value does not match pattern`)
         }
         return sanitize(value, { allowedTags: allowedTags || [] })
       },
     })
+
+    function checkLength(value: any) {
+      if (maxLength && maxLength < value.length) {
+        throw new Error(
+          `Specified value cannot be longer than ${maxLength} characters`
+        )
+      }
+      if (minLength && minLength > value.length) {
+        throw new Error(
+          `Specified value cannot be shorter than ${minLength} characters`
+        )
+      }
+    }
   }
 }
 

packages/api/src/schema.ts

@@ -8,6 +8,7 @@ const schema = gql`
   directive @sanitize(
     allowedTags: [String]
     maxLength: Int
+    minLength: Int
     pattern: String
   ) on INPUT_FIELD_DEFINITION
 
@@ -363,6 +364,7 @@ const schema = gql`
     savedAt: Date!
     updatedAt: Date!
     publishedAt: Date
+    readingProgressTopPercent: Float
     readingProgressPercent: Float!
     readingProgressAnchorIndex: Int!
     sharedComment: String
@@ -565,6 +567,7 @@ const schema = gql`
     description: String
     byline: String
     savedAt: Date
+    publishedAt: Date
   }
 
   type UpdatePageSuccess {
@@ -608,6 +611,7 @@ const schema = gql`
     | SaveArticleReadingProgressError
   input SaveArticleReadingProgressInput {
     id: ID!
+    readingProgressTopPercent: Float
     readingProgressPercent: Float!
     readingProgressAnchorIndex: Int!
   }
@@ -656,18 +660,24 @@ const schema = gql`
     reactions: [Reaction!]!
   }
 
+  enum HighlightType {
+    HIGHLIGHT
+    REDACTION
+    NOTE
+  }
+
   # Highlight
   type Highlight {
     id: ID!
     # used for simplified url format
     shortId: String!
     user: User!
-    quote: String!
+    quote: String
     # piece of content before the quote
     prefix: String
     # piece of content after the quote
     suffix: String
-    patch: String!
+    patch: String
     annotation: String
     replies: [HighlightReply!]!
     sharedAt: Date
@@ -678,20 +688,24 @@ const schema = gql`
     highlightPositionPercent: Float
     highlightPositionAnchorIndex: Int
     labels: [Label!]
+    type: HighlightType!
+    html: String
   }
 
   input CreateHighlightInput {
     id: ID!
     shortId: String!
     articleId: ID!
-    patch: String!
-    quote: String! @sanitize(maxLength: 6000)
+    patch: String
+    quote: String @sanitize(maxLength: 6000, minLength: 1)
     prefix: String @sanitize
     suffix: String @sanitize
     annotation: String @sanitize(maxLength: 4000)
     sharedAt: Date
     highlightPositionPercent: Float
     highlightPositionAnchorIndex: Int
+    type: HighlightType
+    html: String
   }
 
   type CreateHighlightSuccess {
@@ -717,13 +731,14 @@ const schema = gql`
     shortId: ID!
     articleId: ID!
     patch: String!
-    quote: String! @sanitize(maxLength: 6000)
+    quote: String! @sanitize(maxLength: 6000, minLength: 1)
     prefix: String @sanitize
     suffix: String @sanitize
     annotation: String @sanitize(maxLength: 8000)
     overlapHighlightIdList: [String!]!
     highlightPositionPercent: Float
     highlightPositionAnchorIndex: Int
+    html: String
   }
 
   type MergeHighlightSuccess {
@@ -747,8 +762,10 @@ const schema = gql`
 
   input UpdateHighlightInput {
     highlightId: ID!
-    annotation: String @sanitize
+    annotation: String @sanitize(maxLength: 4000)
     sharedAt: Date
+    quote: String @sanitize(maxLength: 6000, minLength: 1)
+    html: String
   }
 
   type UpdateHighlightSuccess {
@@ -1065,6 +1082,7 @@ const schema = gql`
     errorCode: CreateArticleErrorCode
     createdAt: Date!
     updatedAt: Date!
+    url: String!
   }
 
   # Query: ArticleSavingRequest
@@ -1074,6 +1092,7 @@ const schema = gql`
   enum ArticleSavingRequestErrorCode {
     UNAUTHORIZED
     NOT_FOUND
+    BAD_DATA
   }
   type ArticleSavingRequestError {
     errorCodes: [ArticleSavingRequestErrorCode!]!
@@ -1524,6 +1543,7 @@ const schema = gql`
     createdAt: Date!
     updatedAt: Date
     isArchived: Boolean!
+    readingProgressTopPercent: Float
     readingProgressPercent: Float!
     readingProgressAnchorIndex: Int!
     author: String
@@ -2515,7 +2535,7 @@ const schema = gql`
     getFollowers(userId: ID): GetFollowersResult!
     getFollowing(userId: ID): GetFollowingResult!
     getUserPersonalization: GetUserPersonalizationResult!
-    articleSavingRequest(id: ID!): ArticleSavingRequestResult!
+    articleSavingRequest(id: ID, url: String): ArticleSavingRequestResult!
     newsletterEmails: NewsletterEmailsResult!
     reminder(linkId: ID!): ReminderResult!
     labels: LabelsResult!

packages/api/src/server.ts

@@ -7,7 +7,7 @@ import { json, urlencoded } from 'body-parser'
 import cookieParser from 'cookie-parser'
 import express, { Express } from 'express'
 import { createServer, Server } from 'http'
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { env } from './env'
 import * as Sentry from '@sentry/node'
 import * as lw from '@google-cloud/logging-winston'

packages/api/src/services/create_page_save_request.ts

@@ -1,18 +1,22 @@
+import normalizeUrl from 'normalize-url'
+import * as privateIpLib from 'private-ip'
 import { v4 as uuidv4 } from 'uuid'
-import { enqueueParseRequest } from '../utils/createTask'
-
-// TODO: switch to a proper Entity instead of using the old data models.
-import { DataModels } from '../resolvers/types'
+import { createPubSubClient, PubsubClient } from '../datalayer/pubsub'
+import {
+  countByCreatedAt,
+  createPage,
+  getPageByParam,
+  updatePage,
+} from '../elastic/pages'
+import { ArticleSavingRequestStatus, PageType } from '../elastic/types'
 import {
   ArticleSavingRequest,
   CreateArticleSavingRequestErrorCode,
 } from '../generated/graphql'
+// TODO: switch to a proper Entity instead of using the old data models.
+import { DataModels } from '../resolvers/types'
+import { enqueueParseRequest } from '../utils/createTask'
 import { generateSlug, pageToArticleSavingRequest } from '../utils/helpers'
-import * as privateIpLib from 'private-ip'
-import { countByCreatedAt, createPage, getPageByParam } from '../elastic/pages'
-import { ArticleSavingRequestStatus, PageType } from '../elastic/types'
-import { createPubSubClient, PubsubClient } from '../datalayer/pubsub'
-import normalizeUrl from 'normalize-url'
 
 const SAVING_CONTENT = 'Your link is being saved...'
 
@@ -88,14 +92,16 @@ export const createPageSaveRequest = async (
     stripWWW: false,
   })
 
+  const ctx = {
+    pubsub,
+    uid: userId,
+  }
   let page = await getPageByParam({
     userId,
     url: normalizedUrl,
   })
-  if (page) {
-    console.log('Page already exists', page.id, page.url)
-    articleSavingRequestId = page.id
-  } else {
+  if (!page) {
+    console.log('Page not exists', normalizedUrl)
     page = {
       id: articleSavingRequestId,
       userId,
@@ -106,14 +112,14 @@ export const createPageSaveRequest = async (
       readingProgressPercent: 0,
       slug: generateSlug(url),
       title: url,
-      url,
+      url: normalizedUrl,
       state: ArticleSavingRequestStatus.Processing,
       createdAt: new Date(),
       savedAt: new Date(),
     }
 
     // create processing page
-    const pageId = await createPage(page, { pubsub, uid: userId })
+    const pageId = await createPage(page, ctx)
     if (!pageId) {
       console.log('Failed to create page', page)
       return Promise.reject({
@@ -121,9 +127,18 @@ export const createPageSaveRequest = async (
       })
     }
   }
-
+  // reset state to processing
+  if (page.state !== ArticleSavingRequestStatus.Processing) {
+    await updatePage(
+      page.id,
+      {
+        state: ArticleSavingRequestStatus.Processing,
+      },
+      ctx
+    )
+  }
   // enqueue task to parse page
-  await enqueueParseRequest(url, userId, articleSavingRequestId, priority)
+  await enqueueParseRequest(url, userId, page.id, priority)
 
   return pageToArticleSavingRequest(user, page)
 }

packages/api/src/services/create_user.ts

@@ -1,14 +1,14 @@
-import { AuthProvider } from '../routers/auth/auth_types'
-import { StatusType } from '../datalayer/user/model'
 import { EntityManager } from 'typeorm'
-import { User } from '../entity/user'
-import { Profile } from '../entity/profile'
-import { SignupErrorCode } from '../generated/graphql'
-import { validateUsername } from '../utils/usernamePolicy'
-import { Invite } from '../entity/groups/invite'
+import { StatusType } from '../datalayer/user/model'
 import { GroupMembership } from '../entity/groups/group_membership'
-import { AppDataSource } from '../server'
+import { Invite } from '../entity/groups/invite'
+import { Profile } from '../entity/profile'
+import { User } from '../entity/user'
 import { getRepository } from '../entity/utils'
+import { SignupErrorCode } from '../generated/graphql'
+import { AuthProvider } from '../routers/auth/auth_types'
+import { AppDataSource } from '../server'
+import { validateUsername } from '../utils/usernamePolicy'
 import { sendConfirmationEmail } from './send_emails'
 
 export const createUser = async (input: {
@@ -24,7 +24,7 @@ export const createUser = async (input: {
   password?: string
   pendingConfirmation?: boolean
 }): Promise<[User, Profile]> => {
-  const existingUser = await getUser(input.email)
+  const existingUser = await getUserByEmail(input.email)
   if (existingUser) {
     if (existingUser.profile) {
       return Promise.reject({ errorCode: SignupErrorCode.UserExists })
@@ -114,11 +114,10 @@ const validateInvite = async (
   return true
 }
 
-const getUser = async (email: string): Promise<User | null> => {
-  const userRepo = getRepository(User)
-
-  return userRepo.findOne({
-    where: { email: email },
-    relations: ['profile'],
-  })
+export const getUserByEmail = async (email: string): Promise<User | null> => {
+  return getRepository(User)
+    .createQueryBuilder('user')
+    .leftJoinAndSelect('user.profile', 'profile')
+    .where('LOWER(email) = LOWER(:email)', { email }) // case insensitive
+    .getOne()
 }

packages/api/src/services/integrations.ts

@@ -2,7 +2,7 @@ import { IntegrationType } from '../generated/graphql'
 import { env } from '../env'
 import axios from 'axios'
 import { wait } from '../utils/helpers'
-import { Page } from '../elastic/types'
+import { HighlightType, Page } from '../elastic/types'
 import { getHighlightUrl } from './highlights'
 import { Integration } from '../entity/integration'
 import { getRepository } from '../entity/utils'
@@ -65,22 +65,31 @@ const validateReadwiseToken = async (token: string): Promise<boolean> => {
 
 const pageToReadwiseHighlight = (page: Page): ReadwiseHighlight[] => {
   if (!page.highlights) return []
-  return page.highlights.map((highlight) => {
-    return {
-      text: highlight.quote,
-      title: page.title,
-      author: page.author || undefined,
-      highlight_url: getHighlightUrl(page.slug, highlight.id),
-      highlighted_at: new Date(highlight.createdAt).toISOString(),
-      category: 'articles',
-      image_url: page.image || undefined,
-      location: highlight.highlightPositionPercent || undefined,
-      location_type: 'order',
-      note: highlight.annotation || undefined,
-      source_type: 'omnivore',
-      source_url: page.url,
-    }
-  })
+  const category = page.siteName === 'Twitter' ? 'tweets' : 'articles'
+  return (
+    page.highlights
+      // filter out highlights with no quote and are not of type Highlight
+      .filter(
+        (highlight) =>
+          highlight.type === HighlightType.Highlight && highlight.quote
+      )
+      .map((highlight) => {
+        return {
+          text: highlight.quote!,
+          title: page.title,
+          author: page.author || undefined,
+          highlight_url: getHighlightUrl(page.slug, highlight.id),
+          highlighted_at: new Date(highlight.createdAt).toISOString(),
+          category,
+          image_url: page.image || undefined,
+          // location: highlight.highlightPositionAnchorIndex || undefined,
+          location_type: 'order',
+          note: highlight.annotation || undefined,
+          source_type: 'omnivore',
+          source_url: page.url,
+        }
+      })
+  )
 }
 
 export const syncWithIntegration = async (
@@ -131,19 +140,31 @@ export const syncWithReadwise = async (
     )
     return response.status === 200
   } catch (error) {
-    if (
-      axios.isAxiosError(error) &&
-      error.response?.status === 429 &&
-      retryCount < 3
-    ) {
-      console.log('Readwise API rate limit exceeded, retrying...')
-      // wait for Retry-After seconds in the header if rate limited
-      // max retry count is 3
-      const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds
-      await wait(parseInt(retryAfter, 10) * 1000)
-      return syncWithReadwise(token, highlights, retryCount + 1)
+    if (axios.isAxiosError(error)) {
+      if (error.response) {
+        if (error.response.status === 429 && retryCount < 3) {
+          console.log('Readwise API rate limit exceeded, retrying...')
+          // wait for Retry-After seconds in the header if rate limited
+          // max retry count is 3
+          const retryAfter = error.response?.headers['retry-after'] || '10' // default to 10 seconds
+          await wait(parseInt(retryAfter, 10) * 1000)
+          return syncWithReadwise(token, highlights, retryCount + 1)
+        }
+        // The request was made and the server responded with a status code
+        // that falls out of the range of 2xx
+        console.error('Readwise error, response data', error.response.data)
+      } else if (error.request) {
+        // The request was made but no response was received
+        // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
+        // http.ClientRequest in node.js
+        console.error('Readwise error, request', error.request)
+      } else {
+        // Something happened in setting up the request that triggered an Error
+        console.error('Error', error.message)
+      }
+    } else {
+      console.error('Error syncing with readwise', error)
     }
-    console.log('Error creating highlights in Readwise', error)
     return false
   }
 }

packages/api/src/services/save_file.ts

@@ -1,4 +1,4 @@
-import Knex from 'knex'
+import { Knex } from 'knex'
 import { PubsubClient } from '../datalayer/pubsub'
 import { UserData } from '../datalayer/user/model'
 import { homePageURL } from '../env'
@@ -34,12 +34,9 @@ export const saveFile = async (
     }
   }
 
-  const uploadFileDetails = await getStorageFileDetails(
-    input.uploadFileId,
-    uploadFile.fileName
-  )
+  await getStorageFileDetails(input.uploadFileId, uploadFile.fileName)
 
-  const uploadFileData = await ctx.authTrx(async (tx) => {
+  await ctx.authTrx(async (tx) => {
     return ctx.models.uploadFile.setFileUploadComplete(input.uploadFileId, tx)
   })
 

packages/api/src/services/save_page.ts

@@ -1,6 +1,12 @@
+import { Readability } from '@omnivore/readability'
+import normalizeUrl from 'normalize-url'
 import { PubsubClient } from '../datalayer/pubsub'
+import { addHighlightToPage } from '../elastic/highlights'
+import { createPage, getPageByParam, updatePage } from '../elastic/pages'
+import { ArticleSavingRequestStatus, Page, PageType } from '../elastic/types'
 import { homePageURL } from '../env'
 import {
+  HighlightType,
   Maybe,
   PreparedDocumentInput,
   SaveErrorCode,
@@ -15,13 +21,7 @@ import {
   wordsCount,
 } from '../utils/helpers'
 import { parsePreparedContent } from '../utils/parser'
-
-import normalizeUrl from 'normalize-url'
 import { createPageSaveRequest } from './create_page_save_request'
-import { ArticleSavingRequestStatus, Page, PageType } from '../elastic/types'
-import { createPage, getPageByParam, updatePage } from '../elastic/pages'
-import { addHighlightToPage } from '../elastic/highlights'
-import { Readability } from '@omnivore/readability'
 
 type SaveContext = {
   pubsub: PubsubClient
@@ -76,7 +76,6 @@ export const savePage = async (
   saver: SaverUserData,
   input: SavePageInput
 ): Promise<SaveResult> => {
-  const [slug, croppedPathname] = createSlug(input.url, input.title)
   const parseResult = await parsePreparedContent(
     input.url,
     {
@@ -88,12 +87,14 @@ export const savePage = async (
     },
     input.parseResult
   )
-
+  const [newSlug, croppedPathname] = createSlug(input.url, input.title)
+  let slug = newSlug
+  let pageId = input.clientRequestId
   const articleToSave = parsedContentToPage({
     url: input.url,
     title: input.title,
     userId: saver.userId,
-    pageId: input.clientRequestId,
+    pageId,
     slug,
     croppedPathname,
     parsedContent: parseResult.parsedContent,
@@ -101,22 +102,24 @@ export const savePage = async (
     originalHtml: parseResult.domContent,
     canonicalUrl: parseResult.canonicalUrl,
   })
-
-  let pageId: string | undefined = undefined
+  // check if the page already exists
   const existingPage = await getPageByParam({
     userId: saver.userId,
     url: articleToSave.url,
-    state: ArticleSavingRequestStatus.Succeeded,
   })
-
   if (existingPage) {
     pageId = existingPage.id
+    slug = existingPage.slug
     if (
       !(await updatePage(
         existingPage.id,
         {
-          savedAt: new Date(),
-          archivedAt: null,
+          // update the page with the new content
+          ...articleToSave,
+          archivedAt: null, // unarchive if it was archived
+          id: pageId, // we don't want to update the id
+          slug, // we don't want to update the slug
+          createdAt: existingPage.createdAt, // we don't want to update the createdAt
         },
         ctx
       ))
@@ -126,12 +129,11 @@ export const savePage = async (
         message: 'Failed to update existing page',
       }
     }
-    input.clientRequestId = existingPage.id
   } else if (shouldParseInBackend(input)) {
     try {
       await createPageSaveRequest(
         saver.userId,
-        input.url,
+        articleToSave.url,
         ctx.models,
         ctx.pubsub,
         input.clientRequestId
@@ -143,22 +145,24 @@ export const savePage = async (
       }
     }
   } else {
-    pageId = await createPage(articleToSave, ctx)
-    if (!pageId) {
+    const newPageId = await createPage(articleToSave, ctx)
+    if (!newPageId) {
       return {
         errorCodes: [SaveErrorCode.Unknown],
         message: 'Failed to create new page',
       }
     }
+    pageId = newPageId
   }
 
-  if (pageId && parseResult.highlightData) {
+  if (parseResult.highlightData) {
     const highlight = {
       updatedAt: new Date(),
       createdAt: new Date(),
       userId: ctx.uid,
       elasticPageId: pageId,
       ...parseResult.highlightData,
+      type: HighlightType.Highlight,
     }
 
     if (
@@ -175,7 +179,7 @@ export const savePage = async (
   }
 
   return {
-    clientRequestId: input.clientRequestId,
+    clientRequestId: pageId,
     url: `${homePageURL()}/${saver.username}/${slug}`,
   }
 }
@@ -235,7 +239,7 @@ export const parsedContentToPage = ({
     hash: uploadFileHash || stringToHash(parsedContent?.content || url),
     image: parsedContent?.previewImage ?? undefined,
     publishedAt: validatedDate(parsedContent?.publishedDate ?? undefined),
-    uploadFileId: uploadFileId,
+    uploadFileId,
     readingProgressPercent: 0,
     readingProgressAnchorIndex: 0,
     state: ArticleSavingRequestStatus.Succeeded,

packages/api/src/utils/search.ts

@@ -36,6 +36,8 @@ export interface SearchFilter {
   matchFilters: FieldFilter[]
   ids: string[]
   recommendedBy?: string
+  noFilters: NoFilter[]
+  siteName?: string
 }
 
 export enum LabelFilterType {
@@ -83,6 +85,10 @@ export interface FieldFilter {
   value: string
 }
 
+export interface NoFilter {
+  field: string
+}
+
 const parseRecommendedBy = (str?: string): string | undefined => {
   if (str === undefined) {
     return undefined
@@ -263,6 +269,22 @@ const parseIds = (field: string, str?: string): string[] | undefined => {
   return str.split(',')
 }
 
+const parseNoFilter = (str?: string): NoFilter | undefined => {
+  if (str === undefined) {
+    return undefined
+  }
+
+  const strLower = str.toLowerCase()
+  const accepted = ['highlight', 'label']
+  if (accepted.includes(strLower)) {
+    return {
+      field: `${strLower}s`,
+    }
+  }
+
+  return undefined
+}
+
 export const parseSearchQuery = (query: string | undefined): SearchFilter => {
   const searchQuery = query ? query.replace(/\W\s":/g, '') : undefined
   const result: SearchFilter = {
@@ -275,6 +297,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
     termFilters: [],
     matchFilters: [],
     ids: [],
+    noFilters: [],
   }
 
   if (!searchQuery) {
@@ -288,6 +311,7 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
       termFilters: [],
       matchFilters: [],
       ids: [],
+      noFilters: [],
     }
   }
 
@@ -310,6 +334,9 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
       'updated',
       'includes',
       'recommendedBy',
+      'no',
+      'mode',
+      'site',
     ],
     tokenize: true,
   })
@@ -395,6 +422,17 @@ export const parseSearchQuery = (query: string | undefined): SearchFilter => {
           result.recommendedBy = parseRecommendedBy(keyword.value)
           break
         }
+        case 'no': {
+          const noFilter = parseNoFilter(keyword.value)
+          noFilter && result.noFilters.push(noFilter)
+          break
+        }
+        case 'mode':
+          // mode is ignored and used only by the frontend
+          break
+        case 'site':
+          result.siteName = keyword.value
+          break
       }
     }
   }

packages/api/src/utils/uploads.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { env } from '../env'
 import { File, GetSignedUrlConfig, Storage } from '@google-cloud/storage'
+import { env } from '../env'
 
 /* On GAE/Prod, we shall rely on default app engine service account credentials.
  * Two changes needed: 1) add default service account to our uploads GCS Bucket
@@ -83,13 +83,6 @@ export const getStorageFileDetails = async (
   id: string,
   fileName: string
 ): Promise<{ md5Hash: string; fileUrl: string }> => {
-  // if (env.dev.isLocal) {
-  //   return {
-  //     md5Hash: 'some_md5_hash',
-  //     fileUrl: 'http://localhost:3000/public/' + id + '/' + fileName,
-  //   }
-  // }
-
   const filePathName = generateUploadFilePathName(id, fileName)
   const file = storage.bucket(bucketName).file(filePathName)
   const [metadata] = await file.getMetadata()

packages/api/test/elastic/highlights.test.ts

@@ -1,6 +1,11 @@
 import 'mocha'
 import { expect } from 'chai'
-import { Highlight, Page, PageContext } from '../../src/elastic/types'
+import {
+  Highlight,
+  HighlightType,
+  Page,
+  PageContext,
+} from '../../src/elastic/types'
 import { createPubSubClient } from '../../src/datalayer/pubsub'
 import { deletePage } from '../../src/elastic/pages'
 import {
@@ -32,6 +37,7 @@ describe('highlights in elastic', () => {
         userId: page.userId,
         createdAt: new Date(),
         updatedAt: new Date(),
+        type: HighlightType.Highlight,
       }
 
       await addHighlightToPage(page.id, highlightData, ctx)

packages/api/test/elastic/labels.test.ts

@@ -3,6 +3,7 @@ import { expect } from 'chai'
 import {
   ArticleSavingRequestStatus,
   Highlight,
+  HighlightType,
   Label,
   Page,
   PageContext,
@@ -123,6 +124,7 @@ describe('labels in elastic', () => {
         userId: page.userId,
         createdAt: new Date(),
         updatedAt: new Date(),
+        type: HighlightType.Highlight,
       }
 
       await addHighlightToPage(page.id, highlightData, ctx)

packages/api/test/resolvers/article.test.ts

@@ -1,31 +1,10 @@
-import { createTestUser, deleteTestUser } from '../db'
-import {
-  createTestElasticPage,
-  generateFakeUuid,
-  graphqlRequest,
-  request,
-} from '../util'
 import * as chai from 'chai'
 import { expect } from 'chai'
-import 'mocha'
-import { User } from '../../src/entity/user'
 import chaiString from 'chai-string'
-import {
-  BulkActionType,
-  SyncUpdatedItemEdge,
-  UpdateReason,
-  UploadFileStatus,
-} from '../../src/generated/graphql'
-import {
-  ArticleSavingRequestStatus,
-  Highlight,
-  Page,
-  PageContext,
-  PageType,
-} from '../../src/elastic/types'
-import { UploadFile } from '../../src/entity/upload_file'
+import 'mocha'
 import { createPubSubClient } from '../../src/datalayer/pubsub'
-import { getRepository } from '../../src/entity/utils'
+import { refreshIndex } from '../../src/elastic'
+import { addHighlightToPage } from '../../src/elastic/highlights'
 import {
   createPage,
   deletePage,
@@ -33,9 +12,31 @@ import {
   getPageById,
   updatePage,
 } from '../../src/elastic/pages'
-import { addHighlightToPage } from '../../src/elastic/highlights'
-import { refreshIndex } from '../../src/elastic'
+import {
+  ArticleSavingRequestStatus,
+  Highlight,
+  HighlightType,
+  Page,
+  PageContext,
+  PageType,
+} from '../../src/elastic/types'
 import { SearchHistory } from '../../src/entity/search_history'
+import { UploadFile } from '../../src/entity/upload_file'
+import { User } from '../../src/entity/user'
+import { getRepository } from '../../src/entity/utils'
+import {
+  BulkActionType,
+  SyncUpdatedItemEdge,
+  UpdateReason,
+  UploadFileStatus,
+} from '../../src/generated/graphql'
+import { createTestUser, deleteTestUser } from '../db'
+import {
+  createTestElasticPage,
+  generateFakeUuid,
+  graphqlRequest,
+  request,
+} from '../util'
 
 chai.use(chaiString)
 
@@ -298,7 +299,8 @@ const setBookmarkQuery = (articleId: string, bookmark: boolean) => {
 
 const saveArticleReadingProgressQuery = (
   articleId: string,
-  progress: number
+  progress: number,
+  topPercent: number | null = null
 ) => {
   return `
     mutation {
@@ -307,6 +309,7 @@ const saveArticleReadingProgressQuery = (
           id: "${articleId}",
           readingProgressPercent: ${progress}
           readingProgressAnchorIndex: 0
+          readingProgressTopPercent: ${topPercent}
         }
       ) {
         ... on SaveArticleReadingProgressSuccess {
@@ -314,6 +317,7 @@ const saveArticleReadingProgressQuery = (
             id
             readingProgressPercent
             readAt
+            readingProgressTopPercent
           }
         }
         ... on SaveArticleReadingProgressError {
@@ -473,6 +477,7 @@ describe('Article API', () => {
             quote: 'test quote',
             updatedAt: new Date(),
             userId: user.id,
+            type: HighlightType.Highlight,
           },
         ],
       }
@@ -695,9 +700,9 @@ describe('Article API', () => {
 
   describe('saveArticleReadingProgressResolver', () => {
     let query = ''
-    let articleId = ''
-    let progress = 0.5
     let pageId = ''
+    let progress = 0.5
+    let topPercent: number | null = null
 
     before(async () => {
       pageId = (await createTestElasticPage(user.id)).id!
@@ -707,46 +712,71 @@ describe('Article API', () => {
       await deletePage(pageId, ctx)
     })
 
-    beforeEach(() => {
-      query = saveArticleReadingProgressQuery(articleId, progress)
+    it('saves a reading progress on an article', async () => {
+      query = saveArticleReadingProgressQuery(pageId, progress, topPercent)
+      const res = await graphqlRequest(query, authToken).expect(200)
+      expect(
+        res.body.data.saveArticleReadingProgress.updatedArticle
+          .readingProgressPercent
+      ).to.eq(progress)
+      expect(res.body.data.saveArticleReadingProgress.updatedArticle.readAt).not
+        .null
     })
 
-    context('when we save a reading progress on an article', () => {
-      before(async () => {
-        articleId = pageId
-        progress = 0.5
-      })
+    it('should not allow setting the reading progress lower than current progress', async () => {
+      const firstQuery = saveArticleReadingProgressQuery(pageId, 75)
+      const firstRes = await graphqlRequest(firstQuery, authToken).expect(200)
+      expect(
+        firstRes.body.data.saveArticleReadingProgress.updatedArticle
+          .readingProgressPercent
+      ).to.eq(75)
+      await refreshIndex()
 
-      it('should save a reading progress on an article', async () => {
-        const res = await graphqlRequest(query, authToken).expect(200)
-        expect(
-          res.body.data.saveArticleReadingProgress.updatedArticle
-            .readingProgressPercent
-        ).to.eq(progress)
-        expect(res.body.data.saveArticleReadingProgress.updatedArticle.readAt)
-          .not.null
-      })
+      // Now try to set to a lower value (50), value should not be updated
+      // refresh index to ensure the reading progress is updated
+      const secondQuery = saveArticleReadingProgressQuery(pageId, 50)
+      const secondRes = await graphqlRequest(secondQuery, authToken).expect(200)
+      expect(
+        secondRes.body.data.saveArticleReadingProgress.updatedArticle
+          .readingProgressPercent
+      ).to.eq(75)
+    })
 
-      it('should not allow setting the reading progress lower than current progress', async () => {
-        const firstQuery = saveArticleReadingProgressQuery(articleId, 75)
-        const firstRes = await graphqlRequest(firstQuery, authToken).expect(200)
-        expect(
-          firstRes.body.data.saveArticleReadingProgress.updatedArticle
-            .readingProgressPercent
-        ).to.eq(75)
-        await refreshIndex()
+    it('does not save topPercent if not undefined', async () => {
+      query = saveArticleReadingProgressQuery(pageId, progress, null)
+      const res = await graphqlRequest(query, authToken).expect(200)
+      expect(
+        res.body.data.saveArticleReadingProgress.updatedArticle
+          .readingProgressTopPercent
+      ).to.be.null
+    })
 
-        // Now try to set to a lower value (50), value should not be updated
-        // refresh index to ensure the reading progress is updated
-        const secondQuery = saveArticleReadingProgressQuery(articleId, 50)
-        const secondRes = await graphqlRequest(secondQuery, authToken).expect(
-          200
-        )
-        expect(
-          secondRes.body.data.saveArticleReadingProgress.updatedArticle
-            .readingProgressPercent
-        ).to.eq(75)
-      })
+    it('saves topPercent if defined', async () => {
+      const topPercent = 0.2
+      query = saveArticleReadingProgressQuery(pageId, progress, topPercent)
+      const res = await graphqlRequest(query, authToken).expect(200)
+      expect(
+        res.body.data.saveArticleReadingProgress.updatedArticle
+          .readingProgressTopPercent
+      ).to.eql(topPercent)
+    })
+
+    it('saves topPercent as 0 if defined as 0', async () => {
+      const topPercent = 0
+      query = saveArticleReadingProgressQuery(pageId, progress, topPercent)
+      const res = await graphqlRequest(query, authToken).expect(200)
+      expect(
+        res.body.data.saveArticleReadingProgress.updatedArticle
+          .readingProgressTopPercent
+      ).to.eql(topPercent)
+    })
+
+    it('returns BAD_DATA error if top position is greater than bottom position', async () => {
+      query = saveArticleReadingProgressQuery(pageId, 0.5, 0.8)
+      const res = await graphqlRequest(query, authToken).expect(200)
+      expect(res.body.data.saveArticleReadingProgress.errorCodes).to.eql([
+        'BAD_DATA',
+      ])
     })
   })
 
@@ -810,7 +840,7 @@ describe('Article API', () => {
           userId: user.id,
           pageType: PageType.Article,
           title: 'test title',
-          content: '<p>search page</p>',
+          content: '<p>test search api</p>',
           slug: 'test slug',
           createdAt: new Date(),
           updatedAt: new Date(),
@@ -819,6 +849,7 @@ describe('Article API', () => {
           url: url,
           savedAt: new Date(),
           state: ArticleSavingRequestStatus.Succeeded,
+          siteName: 'Example',
         }
         page.id = (await createPage(page, ctx))!
         pages.push(page)
@@ -832,6 +863,7 @@ describe('Article API', () => {
           quote: '<p>search highlight</p>',
           createdAt: new Date(),
           updatedAt: new Date(),
+          type: HighlightType.Highlight,
         }
         await addHighlightToPage(page.id, highlight, ctx)
         highlights.push(highlight)
@@ -849,7 +881,7 @@ describe('Article API', () => {
 
     context('when we search for a keyword', () => {
       before(() => {
-        keyword = 'search'
+        keyword = 'search api'
       })
 
       it('saves the term in search history', async () => {
@@ -875,7 +907,7 @@ describe('Article API', () => {
 
     context('when type:highlights is not in the query', () => {
       before(() => {
-        keyword = 'search'
+        keyword = 'search api'
       })
 
       it('should return pages in descending order', async () => {
@@ -901,7 +933,7 @@ describe('Article API', () => {
 
     context('when type:highlights is in the query', () => {
       before(() => {
-        keyword = 'search type:highlights'
+        keyword = "'search api' type:highlights"
       })
 
       it('should return highlights in descending order', async () => {
@@ -918,7 +950,7 @@ describe('Article API', () => {
 
     context('when is:unread is in the query', () => {
       before(() => {
-        keyword = 'search is:unread'
+        keyword = "'search api' is:unread"
       })
 
       it('should return unread articles in descending order', async () => {
@@ -932,6 +964,42 @@ describe('Article API', () => {
         expect(res.body.data.search.edges[4].node.id).to.eq(pages[0].id)
       })
     })
+
+    context('when no:label is in the query', () => {
+      before(async () => {
+        keyword = "'search api' no:label"
+      })
+
+      it('returns non-labeled items in descending order', async () => {
+        const res = await graphqlRequest(query, authToken).expect(200)
+
+        expect(res.body.data.search.pageInfo.totalCount).to.eq(5)
+      })
+    })
+
+    context('when no:highlight is in the query', () => {
+      before(async () => {
+        keyword = "'search api' no:highlight"
+      })
+
+      it('returns non-highlighted items in descending order', async () => {
+        const res = await graphqlRequest(query, authToken).expect(200)
+
+        expect(res.body.data.search.pageInfo.totalCount).to.eq(0)
+      })
+    })
+
+    context('when site:${site_name} is in the query', () => {
+      before(async () => {
+        keyword = "'search api' site:example"
+      })
+
+      it('returns items from the site', async () => {
+        const res = await graphqlRequest(query, authToken).expect(200)
+
+        expect(res.body.data.search.pageInfo.totalCount).to.eq(5)
+      })
+    })
   })
 
   describe('TypeaheadSearch API', () => {

packages/api/test/resolvers/article_saving_request.test.ts

@@ -1,22 +1,32 @@
-import { User } from '../../src/entity/user'
+import { expect } from 'chai'
+import 'mocha'
+import sinon from 'sinon'
+import { createPubSubClient } from '../../src/datalayer/pubsub'
+import { deletePagesByParam, getPageByParam } from '../../src/elastic/pages'
 import {
   ArticleSavingRequestStatus,
   PageContext,
 } from '../../src/elastic/types'
-import { createTestUser, deleteTestUser } from '../db'
-import { graphqlRequest, request } from '../util'
-import { createPubSubClient } from '../../src/datalayer/pubsub'
-import { expect } from 'chai'
-import { getPageById } from '../../src/elastic/pages'
+import { User } from '../../src/entity/user'
 import {
   ArticleSavingRequestErrorCode,
   CreateArticleSavingRequestErrorCode,
 } from '../../src/generated/graphql'
-import 'mocha'
+import * as createTask from '../../src/utils/createTask'
+import { createTestUser, deleteTestUser } from '../db'
+import { graphqlRequest, request } from '../util'
 
-const articleSavingRequestQuery = (id: string) => `
+const articleSavingRequestQuery = ({
+  id,
+  url,
+}: {
+  id?: string
+  url?: string
+}) => `
   query {
-    articleSavingRequest(id: "${id}") {
+    articleSavingRequest(id: ${id ? `"${id}"` : null}, url: ${
+  url ? `"${url}"` : null
+}) {
       ... on ArticleSavingRequestSuccess {
         articleSavingRequest {
           id
@@ -39,6 +49,7 @@ const createArticleSavingRequestMutation = (url: string) => `
         articleSavingRequest {
           id
           status
+          url
         }
       }
       ... on CreateArticleSavingRequestError {
@@ -67,11 +78,14 @@ describe('ArticleSavingRequest API', () => {
       refresh: true,
       uid: user.id,
     }
+    sinon.replace(createTask, 'enqueueParseRequest', sinon.fake.resolves(''))
   })
 
   after(async () => {
     // clean up
+    await deletePagesByParam({ userId: user.id }, ctx)
     await deleteTestUser(user.id)
+    sinon.restore()
   })
 
   describe('createArticleSavingRequest', () => {
@@ -87,15 +101,14 @@ describe('ArticleSavingRequest API', () => {
     })
 
     it('creates a page in elastic', async () => {
-      const res = await graphqlRequest(
+      const url = 'https://blog.omnivore.app/1'
+      await graphqlRequest(
         createArticleSavingRequestMutation('https://blog.omnivore.app/1'),
         authToken
       ).expect(200)
 
-      const page = await getPageById(
-        res.body.data.createArticleSavingRequest.articleSavingRequest.id
-      )
-      expect(page?.content).to.eq('Your link is being saved...')
+      const page = await getPageByParam({ url })
+      expect(page?.content).to.eql('Your link is being saved...')
     })
 
     it('returns an error if the url is invalid', async () => {
@@ -111,32 +124,44 @@ describe('ArticleSavingRequest API', () => {
   })
 
   describe('articleSavingRequest', () => {
-    let articleSavingRequestId: string
+    let url: string
+    let id: string
 
     before(async () => {
+      url = 'https://blog.omnivore.app/2'
       // create article saving request
       const res = await graphqlRequest(
-        createArticleSavingRequestMutation('https://blog.omnivore.app/2'),
+        createArticleSavingRequestMutation(url),
         authToken
       ).expect(200)
-      articleSavingRequestId =
-        res.body.data.createArticleSavingRequest.articleSavingRequest.id
+      id = res.body.data.createArticleSavingRequest.articleSavingRequest.id
     })
 
     it('returns the article saving request if exists', async () => {
       const res = await graphqlRequest(
-        articleSavingRequestQuery(articleSavingRequestId),
+        articleSavingRequestQuery({ url }),
         authToken
       ).expect(200)
 
-      expect(res.body.data.articleSavingRequest.articleSavingRequest.id).to.eql(
-        articleSavingRequestId
-      )
+      expect(
+        res.body.data.articleSavingRequest.articleSavingRequest.status
+      ).to.eql(ArticleSavingRequestStatus.Processing)
+    })
+
+    it('returns the article saving request by id', async () => {
+      const res = await graphqlRequest(
+        articleSavingRequestQuery({ id }),
+        authToken
+      ).expect(200)
+
+      expect(
+        res.body.data.articleSavingRequest.articleSavingRequest.status
+      ).to.eql(ArticleSavingRequestStatus.Processing)
     })
 
     it('returns not_found if not exists', async () => {
       const res = await graphqlRequest(
-        articleSavingRequestQuery('invalid-id'),
+        articleSavingRequestQuery({ id: 'invalid-id' }),
         authToken
       ).expect(200)
 

packages/api/test/resolvers/highlight.test.ts

@@ -11,19 +11,19 @@ import 'mocha'
 import { User } from '../../src/entity/user'
 import chaiString from 'chai-string'
 import { createPubSubClient } from '../../src/datalayer/pubsub'
-import { PageContext } from '../../src/elastic/types'
+import { HighlightType, PageContext } from '../../src/elastic/types'
 import { deletePage, updatePage } from '../../src/elastic/pages'
 
 chai.use(chaiString)
 
 const createHighlightQuery = (
-  authToken: string,
   linkId: string,
   highlightId: string,
   shortHighlightId: string,
   highlightPositionPercent = 0.0,
   highlightPositionAnchorIndex = 0,
   annotation = '_annotation',
+  html: string | null = null,
   prefix = '_prefix',
   suffix = '_suffix',
   quote = '_quote',
@@ -43,6 +43,7 @@ const createHighlightQuery = (
         highlightPositionPercent: ${highlightPositionPercent},
         highlightPositionAnchorIndex: ${highlightPositionAnchorIndex}
         annotation: "${annotation}"
+        html: "${html}"
       }
     ) {
       ... on CreateHighlightSuccess {
@@ -51,6 +52,7 @@ const createHighlightQuery = (
           highlightPositionPercent
           highlightPositionAnchorIndex
           annotation
+          html
         }
       }
       ... on CreateHighlightError {
@@ -104,23 +106,29 @@ const mergeHighlightQuery = (
   `
 }
 
-const updateHighlightQuery = (
-  authToken: string,
-  highlightId: string,
-  annotation = '_annotation'
-) => {
+const updateHighlightQuery = ({
+  highlightId,
+  annotation = null,
+  quote = null,
+}: {
+  highlightId: string
+  annotation?: string | null
+  quote?: string | null
+}) => {
   return `
   mutation {
     updateHighlight(
       input: {
         annotation: "${annotation}",
         highlightId: "${highlightId}",
+        quote: "${quote}"
       }
     ) {
       ... on UpdateHighlightSuccess {
         highlight {
           id
           annotation
+          quote
         }
       }
       ... on UpdateHighlightError {
@@ -157,18 +165,20 @@ describe('Highlights API', () => {
   })
 
   context('createHighlightMutation', () => {
-    it('should not fail', async () => {
+    it('does not fail', async () => {
       const highlightId = generateFakeUuid()
       const shortHighlightId = '_short_id'
       const highlightPositionPercent = 35.0
       const highlightPositionAnchorIndex = 15
+      const html = '<p>test</p>'
       const query = createHighlightQuery(
-        authToken,
         pageId,
         highlightId,
         shortHighlightId,
         highlightPositionPercent,
-        highlightPositionAnchorIndex
+        highlightPositionAnchorIndex,
+        '_annotation',
+        html
       )
       const res = await graphqlRequest(query, authToken).expect(200)
 
@@ -179,6 +189,7 @@ describe('Highlights API', () => {
       expect(
         res.body.data.createHighlight.highlight.highlightPositionAnchorIndex
       ).to.eq(highlightPositionAnchorIndex)
+      expect(res.body.data.createHighlight.highlight.html).to.eq(html)
     })
 
     context('when the annotation has HTML reserved characters', () => {
@@ -188,7 +199,6 @@ describe('Highlights API', () => {
         const highlightPositionPercent = 50.0
         const highlightPositionAnchorIndex = 25
         const query = createHighlightQuery(
-          authToken,
           pageId,
           newHighlightId,
           newShortHighlightId,
@@ -211,12 +221,7 @@ describe('Highlights API', () => {
       // create test highlight
       highlightId = generateFakeUuid()
       const shortHighlightId = '_short_id_1'
-      const query = createHighlightQuery(
-        authToken,
-        pageId,
-        highlightId,
-        shortHighlightId
-      )
+      const query = createHighlightQuery(pageId, highlightId, shortHighlightId)
       await graphqlRequest(query, authToken).expect(200)
     })
 
@@ -264,6 +269,7 @@ describe('Highlights API', () => {
               quote: '',
               createdAt: new Date(),
               updatedAt: new Date(),
+              type: HighlightType.Highlight,
             },
           ],
         },
@@ -271,15 +277,30 @@ describe('Highlights API', () => {
       )
     })
 
-    context('when the annotation has HTML reserved characters', () => {
-      it('unescapes the annotation and updates', async () => {
-        const annotation = '> This is a test'
-        const query = updateHighlightQuery(authToken, highlightId, annotation)
-        const res = await graphqlRequest(query, authToken).expect(200)
-        expect(res.body.data.updateHighlight.highlight.annotation).to.eql(
-          '> This is a test'
-        )
+    it('updates the quote when the quote is in HTML format when the annotation has HTML reserved characters', async () => {
+      const quote = '> This is a test'
+      const query = updateHighlightQuery({ highlightId, quote })
+      const res = await graphqlRequest(query, authToken).expect(200)
+      expect(res.body.data.updateHighlight.highlight.quote).to.eql(quote)
+    })
+
+    it('updates the quote when the quote is in plain text format', async () => {
+      const quote = 'This is a test'
+      const query = updateHighlightQuery({ highlightId, quote })
+      const res = await graphqlRequest(query, authToken).expect(200)
+      expect(res.body.data.updateHighlight.highlight.quote).to.eql(quote)
+    })
+
+    it('unescapes the annotation and updates the annotation when the annotation has HTML reserved characters', async () => {
+      const annotation = '> This is a test'
+      const query = updateHighlightQuery({
+        highlightId,
+        annotation,
       })
+      const res = await graphqlRequest(query, authToken).expect(200)
+      expect(res.body.data.updateHighlight.highlight.annotation).to.eql(
+        annotation
+      )
     })
   })
 })

packages/api/test/resolvers/labels.test.ts

@@ -14,7 +14,12 @@ import { Label } from '../../src/entity/label'
 import { expect } from 'chai'
 import 'mocha'
 import { User } from '../../src/entity/user'
-import { Highlight, Page, PageContext } from '../../src/elastic/types'
+import {
+  Highlight,
+  HighlightType,
+  Page,
+  PageContext,
+} from '../../src/elastic/types'
 import { getRepository } from '../../src/entity/utils'
 import { deletePage, getPageById } from '../../src/elastic/pages'
 import { createPubSubClient } from '../../src/datalayer/pubsub'
@@ -287,6 +292,7 @@ describe('Labels API', () => {
             createdAt: new Date(),
             labels: [toDeleteLabel],
             updatedAt: new Date(),
+            type: HighlightType.Highlight,
           }
           await addHighlightToPage(page.id, highlight, ctx)
         })
@@ -596,6 +602,7 @@ describe('Labels API', () => {
           shortId: 'test shortId',
           userId: user.id,
           updatedAt: new Date(),
+          type: HighlightType.Highlight,
         }
         await addHighlightToPage(page.id, highlight, ctx)
         labelIds = [labels[0].id, labels[1].id]
@@ -620,6 +627,7 @@ describe('Labels API', () => {
           shortId: 'test shortId',
           userId: user.id,
           updatedAt: new Date(),
+          type: HighlightType.Highlight,
         }
         await addHighlightToPage(page.id, highlight, ctx)
         labelIds = [generateFakeUuid(), generateFakeUuid()]

packages/api/test/routers/auth.test.ts

@@ -1,22 +1,22 @@
-import { createTestUser, deleteTestUser, updateTestUser } from '../db'
-import { generateFakeUuid, request } from '../util'
-import { StatusType } from '../../src/datalayer/user/model'
-import { getRepository } from '../../src/entity/utils'
-import { User } from '../../src/entity/user'
 import { MailDataRequired } from '@sendgrid/helpers/classes/mail'
+import chai, { expect } from 'chai'
 import sinon from 'sinon'
-import * as util from '../../src/utils/sendEmail'
+import sinonChai from 'sinon-chai'
 import supertest from 'supertest'
+import { StatusType } from '../../src/datalayer/user/model'
+import { searchPages } from '../../src/elastic/pages'
+import { User } from '../../src/entity/user'
+import { getRepository } from '../../src/entity/utils'
+import { AuthProvider } from '../../src/routers/auth/auth_types'
+import { createPendingUserToken } from '../../src/routers/auth/jwt_helpers'
 import {
   comparePassword,
   generateVerificationToken,
   hashPassword,
 } from '../../src/utils/auth'
-import sinonChai from 'sinon-chai'
-import chai, { expect } from 'chai'
-import { searchPages } from '../../src/elastic/pages'
-import { createPendingUserToken } from '../../src/routers/auth/jwt_helpers'
-import { AuthProvider } from '../../src/routers/auth/auth_types'
+import * as util from '../../src/utils/sendEmail'
+import { createTestUser, deleteTestUser, updateTestUser } from '../db'
+import { generateFakeUuid, request } from '../util'
 
 chai.use(sinonChai)
 
@@ -50,7 +50,7 @@ describe('auth router', () => {
       before(() => {
         password = validPassword
         username = 'Some_username'
-        email = `${username}@omnivore.app`
+        email = `${username}@omnivore.app ` // space at the end is intentional
         name = 'Some name'
       })
 
@@ -178,7 +178,7 @@ describe('auth router', () => {
 
     context('when email and password are valid', () => {
       before(() => {
-        email = user.email
+        email = user.email + ' ' // space at the end is intentional
         password = correctPassword
       })
 

packages/api/test/routers/integrations.test.ts

@@ -10,7 +10,12 @@ import { User } from '../../src/entity/user'
 import { createTestUser, deleteTestIntegrations, deleteTestUser } from '../db'
 import { Integration, IntegrationType } from '../../src/entity/integration'
 import { getRepository } from '../../src/entity/utils'
-import { Highlight, Page, PageContext } from '../../src/elastic/types'
+import {
+  Highlight,
+  HighlightType,
+  Page,
+  PageContext,
+} from '../../src/elastic/types'
 import nock from 'nock'
 import { READWISE_API_URL } from '../../src/services/integrations'
 import { addHighlightToPage } from '../../src/elastic/highlights'
@@ -142,6 +147,7 @@ describe('Integrations routers', () => {
               updatedAt: new Date(),
               userId: user.id,
               highlightPositionPercent,
+              type: HighlightType.Highlight,
             }
             await addHighlightToPage(page.id, highlight, ctx)
             // create highlights data for integration request
@@ -155,7 +161,7 @@ describe('Integrations routers', () => {
                   highlighted_at: highlight.createdAt.toISOString(),
                   category: 'articles',
                   image_url: page.image,
-                  location: highlightPositionPercent,
+                  // location: highlightPositionPercent,
                   location_type: 'order',
                   note: highlight.annotation,
                   source_type: 'omnivore',

packages/appreader/src/index.jsx

@@ -52,6 +52,11 @@ const mutation = async (name, input) => {
 const App = () => {
   applyStoredTheme(false)
 
+  document.addEventListener('updateLabels', (event) => {
+    console.log('updating labels: ', event.labels)
+    setLabels(event.labels)
+  })
+
   return (
     <>
       <Box
@@ -59,6 +64,7 @@ const App = () => {
           overflowY: 'auto',
           height: '100%',
           width: '100vw',
+          paddingTop: window.webkit ? 0 : '48px', // add 48px to android only
         }}
       >
         <VStack
@@ -77,7 +83,7 @@ const App = () => {
             maxWidthPercentage={window.maxWidthPercentage}
             lineHeight={window.lineHeight}
             highlightOnRelease={window.highlightOnRelease}
-            highContrastFont={window.prefersHighContrastFont ?? true}
+            highContrastText={window.prefersHighContrastFont ?? true}
             articleMutations={{
               createHighlightMutation: (input) =>
                 mutation('createHighlight', input),

packages/content-handler/src/content-handler.ts

@@ -150,8 +150,8 @@ export abstract class ContentHandler {
     // e.g. List-Unsubscribe: <https://omnivore.com/unsub>, <mailto:unsub@omnivore.com>
     const decoded = rfc2047.decode(unSubHeader)
     return {
-      mailTo: decoded.match(/<(https?:\/\/[^>]*)>/)?.[1],
-      httpUrl: decoded.match(/<mailto:([^>]*)>/)?.[1],
+      httpUrl: decoded.match(/<(https?:\/\/[^>]*)>/)?.[1],
+      mailTo: decoded.match(/<mailto:([^>]*)>/)?.[1],
     }
   }
 

packages/content-handler/src/websites/twitter-handler.ts

@@ -228,23 +228,8 @@ const getTweetIds = async (
 
       const ids: Set<string> = new Set()
 
-      // Find the first Show thread button and click it
-      const showRepliesButton = Array.from(
-        document.querySelectorAll('div[dir="auto"]')
-      )
-        .filter(
-          (node) => node.children[0] && node.children[0].tagName === 'SPAN'
-        )
-        .find((node) => node.children[0].innerHTML === 'Show replies')
-
-      if (showRepliesButton) {
-        ;(showRepliesButton as HTMLElement).click()
-
-        await waitFor(2000)
-      }
-
       const distance = 1080
-      const scrollHeight = document.body.scrollHeight
+      let scrollHeight = document.body.scrollHeight
       let currentHeight = 0
       // keep scrolling until there are no more elements
       while (currentHeight < scrollHeight) {
@@ -269,13 +254,31 @@ const getTweetIds = async (
           const id = match[2]
           const username = match[1]
 
-          // skip non-author replies
-          username === author && ids.add(id)
+          // stop at non-author replies
+          if (username !== author) return Array.from(ids)
+          ids.add(id)
         }
 
         window.scrollBy(0, distance)
         await waitFor(500)
         currentHeight += distance
+
+        // Find the show replies button and click it
+        if (currentHeight >= scrollHeight) {
+          const showRepliesButton = Array.from(
+            document.querySelectorAll('div[dir]')
+          )
+            .filter(
+              (node) => node.children[0] && node.children[0].tagName === 'SPAN'
+            )
+            .find((node) => node.children[0].innerHTML === 'Show replies')
+
+          if (showRepliesButton) {
+            ;(showRepliesButton as HTMLElement).click()
+            await waitFor(1000)
+            scrollHeight = document.body.scrollHeight
+          }
+        }
       }
 
       return Array.from(ids)
@@ -371,6 +374,8 @@ export class TwitterHandler extends ContentHandler {
       <meta property="og:image:secure_url" content="${authorImage}" />
       <meta property="og:title" content="${escapedTitle}" />
       <meta property="og:description" content="${description}" />
+      <meta property="article:published_time" content="${tweetData.created_at}" />
+      <meta property="og:site_name" content="Twitter" />
     </head>
     <body>
       <div>

packages/content-handler/src/websites/youtube-handler.ts

@@ -38,19 +38,21 @@ export class YoutubeHandler extends ContentHandler {
   }
 
   async preHandle(url: string): Promise<PreHandleResult> {
+    const BaseUrl = 'https://www.youtube.com'
+    const embedBaseUrl = 'https://www.youtube.com/embed'
     let urlToEncode: string
     let src: string
     const playlistId = getYoutubePlaylistId(url)
     if (playlistId) {
-      urlToEncode = `https://www.youtube.com/playlist?list=${playlistId}`
-      src = `https://www.youtube.com/embed/videoseries?list=${playlistId}`
+      urlToEncode = `${BaseUrl}/playlist?list=${playlistId}`
+      src = `${embedBaseUrl}/videoseries?list=${playlistId}`
     } else {
       const videoId = getYoutubeVideoId(url)
       if (!videoId) {
         return {}
       }
-      urlToEncode = `https://www.youtube.com/watch?v=${videoId}`
-      src = `https://www.youtube.com/embed/${videoId}`
+      urlToEncode = `${BaseUrl}/watch?v=${videoId}`
+      src = `${embedBaseUrl}/${videoId}`
     }
 
     const oembedUrl =
@@ -72,7 +74,6 @@ export class YoutubeHandler extends ContentHandler {
     const height = 350
     const width = height * ratio
     const authorName = _.escape(oembed.author_name)
-
     const content = `
     <html>
       <head><title>${escapedTitle}</title>
@@ -81,6 +82,7 @@ export class YoutubeHandler extends ContentHandler {
       <meta property="og:title" content="${escapedTitle}" />
       <meta property="og:description" content="" />
       <meta property="og:article:author" content="${authorName}" />
+      <meta property="og:site_name" content="YouTube" />
       </head>
       <body>
       <iframe width="${width}" height="${height}" src="${src}" title="${escapedTitle}" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>

packages/content-handler/test/newsletter.test.ts

@@ -516,4 +516,35 @@ describe('Newsletter email test', () => {
       expect(url1).to.not.eql(url2)
     })
   })
+
+  describe('get unsubscribe from header', () => {
+    const mailTo = 'unsub@omnivore.com'
+    const httpUrl = 'https://omnivore.com/unsubscribe'
+
+    it('returns mail to address if exists', () => {
+      const header = `<https://omnivore.com/unsub>, <mailto:${mailTo}>`
+
+      expect(new GenericHandler().parseUnsubscribe(header).mailTo).to.equal(
+        mailTo
+      )
+    })
+
+    it('returns http url if exists', () => {
+      const header = `<${httpUrl}>`
+
+      expect(new GenericHandler().parseUnsubscribe(header).httpUrl).to.equal(
+        httpUrl
+      )
+    })
+
+    context('when unsubscribe header rfc2047 encoded', () => {
+      it('returns mail to address if exists', () => {
+        const header = `=?us-ascii?Q?=3Cmailto=3A654e9594-184c-4884-8e02-e6e58a3a6871+87e39b3d-c3ca-4be?= =?us-ascii?Q?b-ba4d-977cc2ba61e7+067a353f-f775-4f2c-?= =?us-ascii?Q?a5cc-978df38deeca=40unsub=2Ebeehiiv=2Ecom=3E=2C?= =?us-ascii?Q?_=3Chttps=3A=2F=2Fwww=2Emilkroad=2Ecom=2Fsubscribe=2F87e39b3d-c3ca-4beb-ba4d-97?= =?us-ascii?Q?7cc2ba61e7=2Fmanage=3Fpost=5Fid=3D067a353f-f775?= =?us-ascii?Q?-4f2c-a5cc-978df38deeca=3E?=',`
+
+        expect(new GenericHandler().parseUnsubscribe(header).mailTo).to.equal(
+          '654e9594-184c-4884-8e02-e6e58a3a6871+87e39b3d-c3ca-4beb-ba4d-977cc2ba61e7+067a353f-f775-4f2c-a5cc-978df38deeca@unsub.beehiiv.com'
+        )
+      })
+    })
+  })
 })

packages/cypress/package.json

@@ -7,7 +7,7 @@
   },
   "dependencies": {
     "@testing-library/cypress": "^8.0.2",
-    "cypress": "^10.1.0"
+    "cypress": "^12.7.0"
   },
   "devDependencies": {},
   "volta": {

packages/db/elastic_migrations/index_settings.json

@@ -84,6 +84,10 @@
           "userId": {
             "type": "keyword"
           },
+          "type": {
+            "type": "keyword",
+            "null_value": "HIGHLIGHT"
+          },
           "quote": {
             "type": "text",
             "analyzer": "strip_html_analyzer"
@@ -123,9 +127,16 @@
           },
           "highlightPositionAnchorIndex": {
             "type": "integer"
+          },
+          "html": {
+            "type": "text",
+            "analyzer": "strip_html_analyzer"
           }
         }
       },
+      "readingProgressTopPercent": {
+        "type": "float"
+      },
       "readingProgressPercent": {
         "type": "float"
       },
@@ -206,4 +217,4 @@
       }
     }
   }
-}
+}

packages/db/migrate.ts

@@ -148,41 +148,8 @@ const elasticMigration = esClient.indices
       log('Elastic index mappings updated.')
     })
   })
-  .then(() => {
-    log('Adding default state to pages in elastic...')
-    return esClient
-      .update_by_query({
-        index: INDEX_ALIAS,
-        requests_per_second: 250,
-        scroll_size: 500,
-        timeout: '30m',
-        body: {
-          script: {
-            source: 'ctx._source.state = params.state',
-            lang: 'painless',
-            params: {
-              state: 'SUCCEEDED',
-            },
-          },
-          query: {
-            bool: {
-              must_not: [
-                {
-                  exists: {
-                    field: 'state',
-                  },
-                },
-              ],
-            },
-          },
-        },
-      })
-      .then(() => log('Default state added.'))
-  })
   .catch((error) => {
     log(`${chalk.red('Elastic migration failed: ')}${error.message}`, chalk.red)
-    const { appliedMigrations } = error
-    logAppliedMigrations(appliedMigrations)
     process.exit(1)
   })
 

packages/db/migrations/0111.do.remove_unique_key_on_webhooks.sql

@@ -0,0 +1,9 @@
+-- Type: DO
+-- Name: remove_unique_key_on_webhooks
+-- Description: Remove unique constraint of user_id and event_types on webhooks table
+
+BEGIN;
+
+ALTER TABLE omnivore.webhooks DROP CONSTRAINT webhooks_user_id_event_types_key;
+
+COMMIT;

packages/db/migrations/0111.undo.remove_unique_key_on_webhooks.sql

@@ -0,0 +1,9 @@
+-- Type: UNDO
+-- Name: remove_unique_key_on_webhooks
+-- Description: Remove unique constraint of user_id and event_types on webhooks table
+
+BEGIN;
+
+ALTER TABLE omnivore.webhooks ADD CONSTRAINT webhooks_user_id_event_types_key UNIQUE (user_id, event_types);
+
+COMMIT;

packages/inbound-email-handler/src/index.ts

@@ -3,25 +3,30 @@
 /* eslint-disable @typescript-eslint/no-unsafe-argument */
 /* eslint-disable @typescript-eslint/no-unused-vars */
 
+import { PubSub } from '@google-cloud/pubsub'
+import { handleNewsletter } from '@omnivore/content-handler'
 import * as Sentry from '@sentry/serverless'
+import axios from 'axios'
+import * as jwt from 'jsonwebtoken'
 import parseHeaders from 'parse-headers'
 import * as multipart from 'parse-multipart-data'
+import { promisify } from 'util'
 import {
   handleConfirmation,
   isConfirmationEmail,
   parseUnsubscribe,
 } from './newsletter'
-import { PubSub } from '@google-cloud/pubsub'
 import { handlePdfAttachment } from './pdf'
-import { handleNewsletter } from '@omnivore/content-handler'
-import axios from 'axios'
-import { promisify } from 'util'
-import * as jwt from 'jsonwebtoken'
 
 interface SaveReceivedEmailResponse {
   id: string
 }
 
+interface Envelope {
+  to: string[]
+  from: string
+}
+
 const signToken = promisify(jwt.sign)
 
 const NEWSLETTER_EMAIL_RECEIVED_TOPIC = 'newsletterEmailReceived'
@@ -68,6 +73,16 @@ const saveReceivedEmail = async (
   return response.data as SaveReceivedEmailResponse
 }
 
+export const parsedTo = (parsed: Record<string, string>): string => {
+  // envelope to contains the real recipient email address
+  try {
+    const envelope = JSON.parse(parsed.envelope) as Envelope
+    return envelope.to[0]
+  } catch (err) {
+    return parsed.to
+  }
+}
+
 export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
   async (req, res) => {
     try {
@@ -98,17 +113,12 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
       const subject = parsed['subject']
       const html = parsed['html']
       const text = parsed['text']
-
-      // headers added when forwarding email by some rules in Gmail
-      // e.g. 'X-Forwarded-To: recipient@omnivore.app'
-      const forwardedTo = headers['x-forwarded-to']?.toString().split(',')[0]
+      // if an email is forwarded to the inbox, the to is the forwarding email recipient
+      const to = parsedTo(parsed)
       // x-forwarded-for is a space separated list of email address
       // the first one is the forwarding email sender and the last one is the recipient
       // e.g. 'X-Forwarded-For: sender@omnivore.app recipient@omnivore.app'
       const forwardedFrom = headers['x-forwarded-for']?.toString().split(' ')[0]
-
-      // if an email is forwarded to the inbox, the to is the forwarding email recipient
-      const to = forwardedTo || parsed['to']
       const unSubHeader = headers['list-unsubscribe']?.toString()
 
       const { id: receivedEmailId } = await saveReceivedEmail(to, {
@@ -141,7 +151,6 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
         if (isConfirmationEmail(from, subject)) {
           console.log('handleConfirmation', from)
           await handleConfirmation(to, subject)
-          return res.send('ok')
         }
 
         if (pdfAttachment) {
@@ -187,6 +196,7 @@ export const inboundEmailHandler = Sentry.GCPFunction.wrapHttpFunction(
             receivedEmailId,
           },
         })
+        res.send('ok')
       }
     } catch (e) {
       console.log(e)

packages/inbound-email-handler/test/newsletter.test.ts

@@ -1,5 +1,6 @@
-import 'mocha'
 import { expect } from 'chai'
+import 'mocha'
+import { parsedTo } from '../src'
 import {
   getConfirmationCode,
   isConfirmationEmail,
@@ -90,3 +91,23 @@ describe('Newsletter email test', () => {
     })
   })
 })
+
+describe('parsedTo', () => {
+  it('returns envelope to if exists', () => {
+    const to = 'receipient@inbox.omnivore.app'
+    expect(
+      parsedTo({
+        envelope: `{"to":["${to}"],"from":"sender@omnivore.app"}`,
+      })
+    ).to.equal(to)
+  })
+
+  it('returns parsed to if envelope does not exists', () => {
+    const to = 'receipient@inbox.omnivore.app'
+    expect(
+      parsedTo({
+        to,
+      })
+    ).to.equal(to)
+  })
+})

packages/puppeteer-parse/index.js

@@ -397,15 +397,33 @@ function validateUrlString(url) {
   }
 }
 
+function tryParseUrl(urlStr) {
+  if (!urlStr) {
+    return null;
+  }
+  
+  // a regular expression to match all URLs
+  const regex = /(https?:\/\/[^\s]+)/g;
+  
+  const matches = urlStr.match(regex);
+  
+  if (matches) {
+    return matches[0]; // only return first match
+  } else {
+    return null;
+  }
+}
+
 function getUrl(req) {
   const urlStr = (req.query ? req.query.url : undefined) || (req.body ? req.body.url : undefined);
-  if (!urlStr) {
+  const url = tryParseUrl(urlStr)
+  if (!url) {
     throw new Error('No URL specified');
   }
 
-  validateUrlString(urlStr);
+  validateUrlString(url);
 
-  const parsed = Url.parse(urlStr);
+  const parsed = Url.parse(url);
   return parsed.href;
 }
 
@@ -634,7 +652,15 @@ async function retrieveHtml(page, logRecord) {
         document.getElementById('px-block-form-wrapper')) {
         return 'IS_BLOCKED'
       }
-
+      // check if create_time is defined
+      if (typeof create_time !== 'undefined' && create_time) {
+        // create_time is a global variable set by WeChat when rendering the page
+        const date = new Date(create_time * 1000);
+        const dateNode = document.createElement('div');
+        dateNode.className = 'omnivore-published-date';
+        dateNode.innerHTML = date.toLocaleString();
+        document.body.appendChild(dateNode);
+      }
       return document.documentElement.outerHTML;
     }, iframes);
     logRecord.puppeteerSuccess = true;

packages/readabilityjs/Readability.js

@@ -169,9 +169,9 @@ Readability.prototype = {
     lazyLoadingElements: /\S*loading\S*/i,
     // NOTE: These two regular expressions are duplicated in
     // Readability-readerable.js. Please keep both copies in sync.
-    articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist/i,
-    articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*/i,
-    unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped|dvz-social-media-buttons|post-toc|mobile-menu|mobile-navbar/i,
+    articleNegativeLookBehindCandidates: /breadcrumbs|breadcrumb|utils|trilist|_header/i,
+    articleNegativeLookAheadCandidates: /outstream(.?)_|sub(.?)_|m_|omeda-promo-|in-article-advert|block-ad-.*|tl_/i,
+    unlikelyCandidates: /\bad\b|ai2html|banner|breadcrumbs|breadcrumb|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager(?!ow)|popup|yom-remote|copyright|keywords|outline|infinite-list|beta|recirculation|site-index|hide-for-print|post-end-share-cta|post-end-cta-full|post-footer|post-head|post-tag|li-date|main-navigation|programtic-ads|outstream_article|hfeed|comment-holder|back-to-top|show-up-next|onward-journey|topic-tracker|list-nav|block-ad-entity|adSpecs|gift-article-button|modal-title|in-story-masthead|share-tools|standard-dock|expanded-dock|margins-h|subscribe-dialog|icon|bumped|dvz-social-media-buttons|post-toc|mobile-menu|mobile-navbar|tl_article_header/i,
     // okMaybeItsACandidate: /and|article(?!-breadcrumb)|body|column|content|main|shadow|post-header/i,
     get okMaybeItsACandidate() {
       return new RegExp(`and|(?<!${this.articleNegativeLookAheadCandidates.source})article(?!-(${this.articleNegativeLookBehindCandidates.source}))|body|column|content|^(?!main-navigation|main-header)main|shadow|post-header|hfeed site|blog-posts hfeed|container-banners|menu-opacity|header-with-anchor-widget`, 'i')
@@ -1055,7 +1055,10 @@ Readability.prototype = {
   _checkPublishedDate: function (node, matchString) {
     // Skipping meta tags
     if (node.tagName.toLowerCase() === 'meta') return
-
+    // return published date if the class name is 'omnivore-published-date' which we added when we scraped the article
+    if (node.className === 'omnivore-published-date' && this._isValidPublishedDate(node.textContent)) {
+      return new Date(node.textContent);
+    }
     // Searching for the real date in the text content
     let dateRegExpFound = this.REGEXPS.DATES_REGEXPS.find(regexp => regexp.test(node.textContent.trim()))
     dateRegExpFound && (dateRegExpFound = dateRegExpFound.exec(node.textContent.trim()))
@@ -1321,8 +1324,8 @@ Readability.prototype = {
         // Add a point for the paragraph itself as a base.
         contentScore += 1;
 
-        // Add points for any commas within this paragraph.
-        contentScore += innerText.split(",").length;
+        // Add points for any commas (including those in CJK language) within this paragraph.
+        contentScore += innerText.split(/[,，、]/g).length;
 
         // For every 100 characters in this paragraph, add another point. Up to 3 points.
         contentScore += Math.min(Math.floor(innerText.length / 100), 3);
@@ -1932,7 +1935,10 @@ Readability.prototype = {
 
     // get site name
     metadata.siteName = jsonld.siteName ||
-      values["og:site_name"] || null;
+      values["og:site_name"] || 
+      values["twitter:site"] ||
+      values["site_name"] ||
+      values["twitter:domain"];
 
     // get website icon
     const siteIcon = this._doc.querySelector(
@@ -2804,6 +2810,22 @@ Readability.prototype = {
           (weight >= 25 && linkDensity > 0.5 && !(node.className === "tweet" && linkDensity === 1)) ||
           ((embedCount === 1 && contentLength < 75) || embedCount > 1))
 
+        // Allow simple lists of images to remain in pages
+        if (isList && haveToRemove) {
+          for (var x = 0; x < node.children.length; x++) {
+            let child = node.children[x];
+            // Don't filter in lists with li's that contain more than one child
+            if (child.children.length > 1) {
+              return haveToRemove;
+            }
+          }
+          var li_count = node.getElementsByTagName("li").length;
+          // Only allow the list to remain if every li contains an image
+          if (img === li_count) {
+            return false;
+          }
+        }
+
         if (haveToRemove) {
           this.log("Cleaning Conditionally", { className: node.className, children: Array.from(node.children).map(ch => ch.tagName) });
         }
@@ -2987,11 +3009,21 @@ Readability.prototype = {
         metadata.excerpt = paragraphs[0].textContent.trim();
       }
     }
+    if (!metadata.siteName) {
+      // Fallback to hostname
+      try {
+        const host = new URL(this._baseURI).hostname;
+        metadata.siteName = host.replace(/^www\./, "");
+      } catch (e) {
+        // Ignore
+      }
+    }
 
     var textContent = articleContent.textContent;
     return {
       title: this._articleTitle,
-      byline: author,
+      // remove \n and extra spaces and trim the string
+      byline: author ? author.replace(/\n/g, ' ').replace(/\s+/g, ' ').trim() : null,
       dir: this._articleDir,
       content: this._serializer(articleContent),
       textContent: textContent,

packages/readabilityjs/test/index.html

@@ -14,160 +14,22 @@
 <td valign="top" style="width: 250px">
     <ul>
 
-      <li>danwang<br />
-        <a href="./test-pages/danwang/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/danwang/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/danwang/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>omnivore_getting_started<br />
-        <a href="./test-pages/omnivore_getting_started/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/omnivore_getting_started/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/omnivore_getting_started/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>nymag<br />
-        <a href="./test-pages/nymag/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/nymag/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/nymag/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>computerenhance.com<br />
-        <a href="./test-pages/computerenhance.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/computerenhance.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/computerenhance.com/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>substack-michaelshellenberger<br />
-        <a href="./test-pages/substack-michaelshellenberger/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/substack-michaelshellenberger/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/substack-michaelshellenberger/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>stratechery<br />
-        <a href="./test-pages/stratechery/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/stratechery/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/stratechery/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>cnbc<br />
-        <a href="./test-pages/cnbc/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/cnbc/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/cnbc/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>elidourado<br />
-        <a href="./test-pages/elidourado/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/elidourado/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/elidourado/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>financialpost-fishing-for-chips<br />
-        <a href="./test-pages/financialpost-fishing-for-chips/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/financialpost-fishing-for-chips/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/financialpost-fishing-for-chips/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>cavesocial<br />
-        <a href="./test-pages/cavesocial/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/cavesocial/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/cavesocial/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>newsletters<br />
-        <a href="./test-pages/newsletters/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/newsletters/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/newsletters/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>spakhm<br />
-        <a href="./test-pages/spakhm/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/spakhm/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/spakhm/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>nytimes-podcasts<br />
-        <a href="./test-pages/nytimes-podcasts/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/nytimes-podcasts/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/nytimes-podcasts/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>china.substack<br />
-        <a href="./test-pages/china.substack/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/china.substack/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/china.substack/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>community.musictribe.com<br />
-        <a href="./test-pages/community.musictribe.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/community.musictribe.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/community.musictribe.com/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>berthub-2<br />
-        <a href="./test-pages/berthub-2/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/berthub-2/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/berthub-2/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
       <li>samcurry<br />
         <a href="./test-pages/samcurry/source.html" target="iframe_b">[source]</a>
         <a href="./test-pages/samcurry/expected.html" target="iframe_b">[readability]</a>
         <a href="./test-pages/samcurry/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>variety<br />
-        <a href="./test-pages/variety/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/variety/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/variety/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>garymarcus<br />
+        <a href="./test-pages/garymarcus/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/garymarcus/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/garymarcus/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>infoproc<br />
-        <a href="./test-pages/infoproc/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/infoproc/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/infoproc/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>computer.rip<br />
-        <a href="./test-pages/computer.rip/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/computer.rip/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/computer.rip/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>news.utexas<br />
-        <a href="./test-pages/news.utexas/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/news.utexas/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/news.utexas/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>electrek<br />
-        <a href="./test-pages/electrek/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/electrek/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/electrek/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>sciencedirect<br />
-        <a href="./test-pages/sciencedirect/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/sciencedirect/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/sciencedirect/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>jacobbrazeal<br />
-        <a href="./test-pages/jacobbrazeal/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/jacobbrazeal/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/jacobbrazeal/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>getting_started_with_omnivore<br />
-        <a href="./test-pages/getting_started_with_omnivore/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/getting_started_with_omnivore/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/getting_started_with_omnivore/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>berthub<br />
-        <a href="./test-pages/berthub/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/berthub/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/berthub/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>vanityfair<br />
+        <a href="./test-pages/vanityfair/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/vanityfair/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/vanityfair/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
       <li>josephg<br />
@@ -176,28 +38,40 @@
         <a href="./test-pages/josephg/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>gflownet<br />
-        <a href="./test-pages/gflownet/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/gflownet/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/gflownet/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>gdcvault<br />
-        <a href="./test-pages/gdcvault/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/gdcvault/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/gdcvault/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
       <li>instyle<br />
         <a href="./test-pages/instyle/source.html" target="iframe_b">[source]</a>
         <a href="./test-pages/instyle/expected.html" target="iframe_b">[readability]</a>
         <a href="./test-pages/instyle/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>slowboring<br />
-        <a href="./test-pages/slowboring/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/slowboring/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/slowboring/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>substack-michaelshellenberger<br />
+        <a href="./test-pages/substack-michaelshellenberger/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/substack-michaelshellenberger/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/substack-michaelshellenberger/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>community.musictribe.com<br />
+        <a href="./test-pages/community.musictribe.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/community.musictribe.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/community.musictribe.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>omnivore_getting_started<br />
+        <a href="./test-pages/omnivore_getting_started/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/omnivore_getting_started/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/omnivore_getting_started/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>wechat<br />
+        <a href="./test-pages/wechat/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/wechat/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/wechat/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>city-journal<br />
+        <a href="./test-pages/city-journal/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/city-journal/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/city-journal/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
       <li>bookofhook.blogspot.com<br />
@@ -206,28 +80,112 @@
         <a href="./test-pages/bookofhook.blogspot.com/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>dailymail<br />
-        <a href="./test-pages/dailymail/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/dailymail/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/dailymail/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>moxie.org<br />
+        <a href="./test-pages/moxie.org/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/moxie.org/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/moxie.org/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>fiercepharma<br />
-        <a href="./test-pages/fiercepharma/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/fiercepharma/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/fiercepharma/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>getting_started_with_omnivore<br />
+        <a href="./test-pages/getting_started_with_omnivore/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/getting_started_with_omnivore/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/getting_started_with_omnivore/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>nytimes<br />
-        <a href="./test-pages/nytimes/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/nytimes/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/nytimes/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>jsomers<br />
+        <a href="./test-pages/jsomers/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/jsomers/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/jsomers/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>wechat<br />
-        <a href="./test-pages/wechat/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/wechat/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/wechat/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>china.substack<br />
+        <a href="./test-pages/china.substack/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/china.substack/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/china.substack/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>brookings.edu<br />
+        <a href="./test-pages/brookings.edu/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/brookings.edu/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/brookings.edu/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>elidourado<br />
+        <a href="./test-pages/elidourado/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/elidourado/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/elidourado/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>thevaluable.dev<br />
+        <a href="./test-pages/thevaluable.dev/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/thevaluable.dev/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/thevaluable.dev/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>substack-email<br />
+        <a href="./test-pages/substack-email/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/substack-email/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/substack-email/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>fast-company<br />
+        <a href="./test-pages/fast-company/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/fast-company/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/fast-company/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>habr.com<br />
+        <a href="./test-pages/habr.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/habr.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/habr.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>sydney.com<br />
+        <a href="./test-pages/sydney.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/sydney.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/sydney.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>blog.jetbrains.com<br />
+        <a href="./test-pages/blog.jetbrains.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/blog.jetbrains.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/blog.jetbrains.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>aboveavalon<br />
+        <a href="./test-pages/aboveavalon/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/aboveavalon/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/aboveavalon/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>danwang<br />
+        <a href="./test-pages/danwang/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/danwang/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/danwang/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>electrek<br />
+        <a href="./test-pages/electrek/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/electrek/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/electrek/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>yuyue.com<br />
+        <a href="./test-pages/yuyue.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/yuyue.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/yuyue.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>spakhm<br />
+        <a href="./test-pages/spakhm/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/spakhm/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/spakhm/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>newsletters<br />
+        <a href="./test-pages/newsletters/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/newsletters/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/newsletters/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
       <li>milkroad<br />
@@ -242,118 +200,16 @@
         <a href="./test-pages/robinwieruch.de/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>github-blog<br />
-        <a href="./test-pages/github-blog/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/github-blog/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/github-blog/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>variety<br />
+        <a href="./test-pages/variety/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/variety/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/variety/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>johnhcochrane.blogspot<br />
-        <a href="./test-pages/johnhcochrane.blogspot/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/johnhcochrane.blogspot/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/johnhcochrane.blogspot/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>jsomers<br />
-        <a href="./test-pages/jsomers/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/jsomers/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/jsomers/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>medium<br />
-        <a href="./test-pages/medium/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/medium/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/medium/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>erik-engheim<br />
-        <a href="./test-pages/erik-engheim/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/erik-engheim/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/erik-engheim/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>habr.com<br />
-        <a href="./test-pages/habr.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/habr.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/habr.com/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>fast-company<br />
-        <a href="./test-pages/fast-company/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/fast-company/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/fast-company/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>youtube-embed<br />
-        <a href="./test-pages/youtube-embed/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/youtube-embed/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/youtube-embed/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>ottawacitizen.com<br />
-        <a href="./test-pages/ottawacitizen.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/ottawacitizen.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/ottawacitizen.com/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>danluu<br />
-        <a href="./test-pages/danluu/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/danluu/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/danluu/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>substack-email<br />
-        <a href="./test-pages/substack-email/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/substack-email/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/substack-email/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>moxie.org<br />
-        <a href="./test-pages/moxie.org/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/moxie.org/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/moxie.org/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>vanityfair<br />
-        <a href="./test-pages/vanityfair/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/vanityfair/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/vanityfair/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>stackoverflow<br />
-        <a href="./test-pages/stackoverflow/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/stackoverflow/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/stackoverflow/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>blog.jetbrains.com<br />
-        <a href="./test-pages/blog.jetbrains.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/blog.jetbrains.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/blog.jetbrains.com/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>bitfieldconsulting<br />
-        <a href="./test-pages/bitfieldconsulting/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/bitfieldconsulting/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/bitfieldconsulting/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>techcrunch<br />
-        <a href="./test-pages/techcrunch/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/techcrunch/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/techcrunch/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>rootsofprogress<br />
-        <a href="./test-pages/rootsofprogress/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/rootsofprogress/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/rootsofprogress/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>channelnewsasia<br />
-        <a href="./test-pages/channelnewsasia/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/channelnewsasia/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/channelnewsasia/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>infoproc<br />
+        <a href="./test-pages/infoproc/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/infoproc/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/infoproc/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
       <li>jon.bo<br />
@@ -362,40 +218,34 @@
         <a href="./test-pages/jon.bo/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>aboveavalon<br />
-        <a href="./test-pages/aboveavalon/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/aboveavalon/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/aboveavalon/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>berthub<br />
+        <a href="./test-pages/berthub/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/berthub/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/berthub/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>city-journal<br />
-        <a href="./test-pages/city-journal/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/city-journal/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/city-journal/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>cavesocial<br />
+        <a href="./test-pages/cavesocial/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/cavesocial/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/cavesocial/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>energias-renovables.com<br />
-        <a href="./test-pages/energias-renovables.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/energias-renovables.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/energias-renovables.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>github-blog<br />
+        <a href="./test-pages/github-blog/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/github-blog/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/github-blog/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>biospace<br />
-        <a href="./test-pages/biospace/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/biospace/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/biospace/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>news.utexas<br />
+        <a href="./test-pages/news.utexas/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/news.utexas/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/news.utexas/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>guardian<br />
-        <a href="./test-pages/guardian/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/guardian/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/guardian/distiller.html" target="iframe_b">[dom-distiller]</a>
-      </li>
-  
-      <li>thevaluable.dev<br />
-        <a href="./test-pages/thevaluable.dev/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/thevaluable.dev/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/thevaluable.dev/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>rootsofprogress<br />
+        <a href="./test-pages/rootsofprogress/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/rootsofprogress/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/rootsofprogress/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
       <li>mathoverflow<br />
@@ -404,10 +254,28 @@
         <a href="./test-pages/mathoverflow/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>nytimes.com<br />
-        <a href="./test-pages/nytimes.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/nytimes.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/nytimes.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>stratechery<br />
+        <a href="./test-pages/stratechery/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/stratechery/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/stratechery/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>cnbc<br />
+        <a href="./test-pages/cnbc/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/cnbc/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/cnbc/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>erik-engheim<br />
+        <a href="./test-pages/erik-engheim/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/erik-engheim/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/erik-engheim/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>berthub-2<br />
+        <a href="./test-pages/berthub-2/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/berthub-2/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/berthub-2/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
       <li>people<br />
@@ -416,22 +284,34 @@
         <a href="./test-pages/people/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>garymarcus<br />
-        <a href="./test-pages/garymarcus/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/garymarcus/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/garymarcus/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>youtube-embed<br />
+        <a href="./test-pages/youtube-embed/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/youtube-embed/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/youtube-embed/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>ft.com<br />
-        <a href="./test-pages/ft.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/ft.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/ft.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>computer.rip<br />
+        <a href="./test-pages/computer.rip/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/computer.rip/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/computer.rip/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>yuyue.com<br />
-        <a href="./test-pages/yuyue.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/yuyue.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/yuyue.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>nytimes<br />
+        <a href="./test-pages/nytimes/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/nytimes/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/nytimes/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>ahhhhfs.com<br />
+        <a href="./test-pages/ahhhhfs.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/ahhhhfs.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/ahhhhfs.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>computerenhance.com<br />
+        <a href="./test-pages/computerenhance.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/computerenhance.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/computerenhance.com/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
       <li>debugger.medium<br />
@@ -440,22 +320,154 @@
         <a href="./test-pages/debugger.medium/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
+      <li>ottawacitizen.com<br />
+        <a href="./test-pages/ottawacitizen.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/ottawacitizen.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/ottawacitizen.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>gflownet<br />
+        <a href="./test-pages/gflownet/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/gflownet/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/gflownet/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>nytimes.com<br />
+        <a href="./test-pages/nytimes.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/nytimes.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/nytimes.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>channelnewsasia<br />
+        <a href="./test-pages/channelnewsasia/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/channelnewsasia/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/channelnewsasia/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>bitfieldconsulting<br />
+        <a href="./test-pages/bitfieldconsulting/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/bitfieldconsulting/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/bitfieldconsulting/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>stackoverflow<br />
+        <a href="./test-pages/stackoverflow/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/stackoverflow/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/stackoverflow/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>techcrunch<br />
+        <a href="./test-pages/techcrunch/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/techcrunch/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/techcrunch/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>biospace<br />
+        <a href="./test-pages/biospace/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/biospace/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/biospace/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>nytimes-podcasts<br />
+        <a href="./test-pages/nytimes-podcasts/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/nytimes-podcasts/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/nytimes-podcasts/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>medium<br />
+        <a href="./test-pages/medium/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/medium/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/medium/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>dailymail<br />
+        <a href="./test-pages/dailymail/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/dailymail/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/dailymail/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>sciencedirect<br />
+        <a href="./test-pages/sciencedirect/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/sciencedirect/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/sciencedirect/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>ft.com<br />
+        <a href="./test-pages/ft.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/ft.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/ft.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>jacobbrazeal<br />
+        <a href="./test-pages/jacobbrazeal/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/jacobbrazeal/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/jacobbrazeal/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>slowboring<br />
+        <a href="./test-pages/slowboring/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/slowboring/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/slowboring/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>danluu<br />
+        <a href="./test-pages/danluu/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/danluu/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/danluu/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>nymag<br />
+        <a href="./test-pages/nymag/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/nymag/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/nymag/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>energias-renovables.com<br />
+        <a href="./test-pages/energias-renovables.com/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/energias-renovables.com/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/energias-renovables.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>fiercepharma<br />
+        <a href="./test-pages/fiercepharma/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/fiercepharma/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/fiercepharma/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>gdcvault<br />
+        <a href="./test-pages/gdcvault/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/gdcvault/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/gdcvault/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
       <li>channelnewsasia02<br />
         <a href="./test-pages/channelnewsasia02/source.html" target="iframe_b">[source]</a>
         <a href="./test-pages/channelnewsasia02/expected.html" target="iframe_b">[readability]</a>
         <a href="./test-pages/channelnewsasia02/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>brookings.edu<br />
-        <a href="./test-pages/brookings.edu/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/brookings.edu/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/brookings.edu/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>johnhcochrane.blogspot<br />
+        <a href="./test-pages/johnhcochrane.blogspot/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/johnhcochrane.blogspot/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/johnhcochrane.blogspot/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
-      <li>sydney.com<br />
-        <a href="./test-pages/sydney.com/source.html" target="iframe_b">[source]</a>
-        <a href="./test-pages/sydney.com/expected.html" target="iframe_b">[readability]</a>
-        <a href="./test-pages/sydney.com/distiller.html" target="iframe_b">[dom-distiller]</a>
+      <li>financialpost-fishing-for-chips<br />
+        <a href="./test-pages/financialpost-fishing-for-chips/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/financialpost-fishing-for-chips/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/financialpost-fishing-for-chips/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>guardian<br />
+        <a href="./test-pages/guardian/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/guardian/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/guardian/distiller.html" target="iframe_b">[dom-distiller]</a>
+      </li>
+  
+      <li>zhihu<br />
+        <a href="./test-pages/zhihu/source.html" target="iframe_b">[source]</a>
+        <a href="./test-pages/zhihu/expected.html" target="iframe_b">[readability]</a>
+        <a href="./test-pages/zhihu/distiller.html" target="iframe_b">[dom-distiller]</a>
       </li>
   
     </ul>