omnivore/packages/web/lib/networking/queries/useGetLibraryItemsQuery.tsx

457 lines
11 KiB
TypeScript
Raw Normal View History

2022-02-11 17:24:33 +00:00
import { gql } from 'graphql-request'
import useSWRInfinite from 'swr/infinite'
import { gqlFetcher } from '../networkHelpers'
import { PageType, State } from '../fragments/articleFragment'
import { ContentReader } from '../fragments/articleFragment'
2022-02-11 17:24:33 +00:00
import { setLinkArchivedMutation } from '../mutations/setLinkArchivedMutation'
import { deleteLinkMutation } from '../mutations/deleteLinkMutation'
2022-06-01 20:17:34 +00:00
import { unsubscribeMutation } from '../mutations/unsubscribeMutation'
2022-02-11 17:24:33 +00:00
import { articleReadingProgressMutation } from '../mutations/articleReadingProgressMutation'
2022-05-05 01:54:52 +00:00
import { Label } from './../fragments/labelFragment'
import {
showErrorToast,
showSuccessToast,
showSuccessToastWithUndo,
} from '../../toastHelpers'
import { Highlight, highlightFragment } from '../fragments/highlightFragment'
import { updatePageMutation } from '../mutations/updatePageMutation'
2022-02-11 17:24:33 +00:00
export interface ReadableItem {
id: string
title: string
slug: string
}
2022-02-11 17:24:33 +00:00
export type LibraryItemsQueryInput = {
limit: number
sortDescending: boolean
searchQuery?: string
cursor?: string
}
type LibraryItemsQueryResponse = {
itemsPages?: LibraryItemsData[]
itemsDataError?: unknown
2022-02-11 17:24:33 +00:00
isLoading: boolean
isValidating: boolean
error: boolean
2022-02-11 17:24:33 +00:00
size: number
setSize: (
size: number | ((_size: number) => number)
) => Promise<unknown[] | undefined>
performActionOnItem: (action: LibraryItemAction, item: LibraryItem) => void
mutate: () => void
2022-02-11 17:24:33 +00:00
}
type LibraryItemAction =
| 'archive'
| 'unarchive'
| 'delete'
| 'mark-read'
| 'mark-unread'
| 'refresh'
2022-06-01 20:17:34 +00:00
| 'unsubscribe'
2022-06-07 08:40:29 +00:00
| 'update-item'
2022-02-11 17:24:33 +00:00
export type LibraryItemsData = {
search: LibraryItems
2022-02-11 17:24:33 +00:00
}
export type LibraryItems = {
2022-02-11 17:24:33 +00:00
edges: LibraryItem[]
pageInfo: PageInfo
errorCodes?: string[]
}
export type LibraryItem = {
cursor: string
node: LibraryItemNode
isLoading?: boolean | undefined
2022-02-11 17:24:33 +00:00
}
export type LibraryItemNode = {
id: string
title: string
url: string
author?: string
image?: string
createdAt: string
publishedAt?: string
contentReader?: ContentReader
2022-02-11 17:24:33 +00:00
originalArticleUrl: string
readingProgressPercent: number
2023-03-10 08:47:02 +00:00
readingProgressTopPercent?: number
readingProgressAnchorIndex: number
slug: string
isArchived: boolean
description: string
ownedByViewer: boolean
uploadFileId: string
labels?: Label[]
pageId: string
shortId: string
quote: string
annotation: string
2022-05-03 19:09:32 +00:00
state: State
pageType: PageType
2022-05-12 02:31:14 +00:00
siteName?: string
2024-02-29 06:08:01 +00:00
siteIcon?: string
subscription?: string
2022-06-05 12:00:38 +00:00
readAt?: string
2023-01-02 08:12:08 +00:00
savedAt?: string
wordsCount?: number
2024-02-27 10:49:21 +00:00
aiSummary?: string
recommendations?: Recommendation[]
highlights?: Highlight[]
}
export type Recommendation = {
id: string
name: string
note?: string
user?: RecommendingUser
recommendedAt: Date
}
export type RecommendingUser = {
userId: string
name: string
username: string
profileImageURL?: string
2022-02-11 17:24:33 +00:00
}
export type PageInfo = {
hasNextPage: boolean
hasPreviousPage: boolean
startCursor: string
endCursor: string
totalCount: number
}
export const recommendationFragment = gql`
fragment RecommendationFields on Recommendation {
id
name
note
user {
userId
name
username
profileImageURL
}
recommendedAt
}
`
2022-02-11 17:24:33 +00:00
export function useGetLibraryItemsQuery({
limit,
sortDescending,
searchQuery,
cursor,
}: LibraryItemsQueryInput): LibraryItemsQueryResponse {
const query = gql`
query Search($after: String, $first: Int, $query: String) {
search(first: $first, after: $after, query: $query) {
... on SearchSuccess {
2022-02-11 17:24:33 +00:00
edges {
cursor
node {
id
title
slug
url
pageType
contentReader
createdAt
isArchived
readingProgressPercent
2023-03-11 00:09:43 +00:00
readingProgressTopPercent
readingProgressAnchorIndex
author
image
description
publishedAt
ownedByViewer
2022-02-11 17:24:33 +00:00
originalArticleUrl
uploadFileId
labels {
id
name
color
}
pageId
shortId
quote
annotation
2022-05-05 01:54:52 +00:00
state
2022-05-12 02:31:14 +00:00
siteName
2024-02-29 06:08:01 +00:00
siteIcon
2022-06-01 20:17:34 +00:00
subscription
2022-06-05 12:00:38 +00:00
readAt
2023-01-02 08:12:08 +00:00
savedAt
wordsCount
recommendations {
id
name
note
user {
userId
name
username
profileImageURL
}
recommendedAt
}
highlights {
...HighlightFields
}
2022-02-11 17:24:33 +00:00
}
}
pageInfo {
hasNextPage
hasPreviousPage
startCursor
endCursor
totalCount
}
}
... on SearchError {
2022-02-11 17:24:33 +00:00
errorCodes
}
}
}
${highlightFragment}
2022-02-11 17:24:33 +00:00
`
const variables = {
after: cursor,
first: limit,
query: searchQuery,
}
const { data, error, mutate, size, setSize, isValidating } = useSWRInfinite(
(pageIndex, previousPageData) => {
const key = [query, limit, sortDescending, searchQuery, undefined]
const previousResult = previousPageData as LibraryItemsData
if (pageIndex === 0) {
return key
}
return [
query,
limit,
sortDescending,
searchQuery,
pageIndex === 0 ? undefined : previousResult.search.pageInfo.endCursor,
2022-02-11 17:24:33 +00:00
]
},
(_query, _l, _s, _sq, cursor) => {
2022-02-11 17:24:33 +00:00
return gqlFetcher(query, { ...variables, after: cursor }, true)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
},
{ revalidateFirstPage: false }
2022-02-11 17:24:33 +00:00
)
let responseError = error
let responsePages = data as LibraryItemsData[] | undefined
// We need to check the response errors here and return the error
// it will be nested in the data pages, if there is one error,
// we invalidate the data and return the error. We also zero out
// the response in the case of an error.
2022-03-01 22:40:36 +00:00
if (!error && responsePages) {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
const errors = responsePages.filter(
(d) => d.search.errorCodes && d.search.errorCodes.length > 0
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
)
2022-02-11 17:24:33 +00:00
if (errors?.length > 0) {
responseError = errors
responsePages = undefined
}
}
2022-06-07 08:40:29 +00:00
const getIndexOf = (page: LibraryItems, item: LibraryItem) => {
return page.edges.findIndex((i) => i.node.id === item.node.id)
2022-06-07 08:40:29 +00:00
}
2022-02-11 17:24:33 +00:00
const performActionOnItem = async (
action: LibraryItemAction,
item: LibraryItem
) => {
if (!responsePages) {
return
}
const updateData = (mutatedItem: LibraryItem | undefined) => {
if (!responsePages) {
return
}
2022-06-07 08:40:29 +00:00
for (const searchResults of responsePages) {
2022-06-07 08:40:29 +00:00
const itemIndex = getIndexOf(searchResults.search, item)
2022-02-11 17:24:33 +00:00
if (itemIndex !== -1) {
if (typeof mutatedItem === 'undefined') {
searchResults.search.edges.splice(itemIndex, 1)
2022-02-11 17:24:33 +00:00
} else {
2022-06-07 08:40:29 +00:00
searchResults.search.edges.splice(itemIndex, 1, mutatedItem)
2022-02-11 17:24:33 +00:00
}
break
}
}
mutate(responsePages, false)
}
switch (action) {
case 'archive':
if (/in:all/.test(query)) {
updateData({
cursor: item.cursor,
node: {
...item.node,
isArchived: true,
},
})
} else {
updateData(undefined)
}
setLinkArchivedMutation({
linkId: item.node.id,
archived: true,
}).then((res) => {
if (res) {
showSuccessToast('Link archived', { position: 'bottom-right' })
} else {
showErrorToast('Error archiving link', { position: 'bottom-right' })
}
2022-02-11 17:24:33 +00:00
})
break
case 'unarchive':
if (/in:all/.test(query)) {
updateData({
cursor: item.cursor,
node: {
...item.node,
isArchived: false,
},
})
} else {
updateData(undefined)
}
setLinkArchivedMutation({
linkId: item.node.id,
archived: false,
}).then((res) => {
if (res) {
showSuccessToast('Link unarchived', { position: 'bottom-right' })
} else {
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
showErrorToast('Error unarchiving link', {
position: 'bottom-right',
})
}
2022-02-11 17:24:33 +00:00
})
break
case 'delete':
updateData(undefined)
const pageId = item.node.id
deleteLinkMutation(pageId).then((res) => {
if (res) {
showSuccessToastWithUndo('Page deleted', async () => {
const result = await updatePageMutation({
pageId: pageId,
state: State.SUCCEEDED,
})
mutate()
if (result) {
showSuccessToast('Page recovered')
} else {
showErrorToast(
'Error recovering page, check your deleted items'
)
}
})
} else {
showErrorToast('Error removing link', { position: 'bottom-right' })
}
})
2022-02-11 17:24:33 +00:00
break
case 'mark-read':
updateData({
cursor: item.cursor,
node: {
...item.node,
readingProgressPercent: 100,
readingProgressTopPercent: 100,
2022-02-11 17:24:33 +00:00
},
})
articleReadingProgressMutation({
id: item.node.id,
force: true,
2022-02-11 17:24:33 +00:00
readingProgressPercent: 100,
readingProgressTopPercent: 100,
2022-02-11 17:24:33 +00:00
readingProgressAnchorIndex: 0,
})
break
case 'mark-unread':
updateData({
cursor: item.cursor,
node: {
...item.node,
readingProgressPercent: 0,
readingProgressTopPercent: 0,
readingProgressAnchorIndex: 0,
2022-02-11 17:24:33 +00:00
},
})
articleReadingProgressMutation({
id: item.node.id,
force: true,
2022-02-11 17:24:33 +00:00
readingProgressPercent: 0,
readingProgressTopPercent: 0,
2022-02-11 17:24:33 +00:00
readingProgressAnchorIndex: 0,
})
break
// case 'unsubscribe':
// if (!!item.node.subscription) {
// updateData({
// cursor: item.cursor,
// node: {
// ...item.node,
// subscription: undefined,
// },
// })
// unsubscribeMutation(item.node.subscription).then((res) => {
// if (res) {
// showSuccessToast('Unsubscribed successfully', {
// position: 'bottom-right',
// })
// } else {
// showErrorToast('Error unsubscribing', {
// position: 'bottom-right',
// })
// }
// })
// }
case 'update-item':
updateData(item)
2022-06-01 20:17:34 +00:00
break
case 'refresh':
await mutate()
2022-02-11 17:24:33 +00:00
}
}
return {
2022-02-11 17:24:33 +00:00
isValidating,
itemsPages: responsePages || undefined,
itemsDataError: responseError,
2022-02-11 17:24:33 +00:00
isLoading: !error && !data,
performActionOnItem,
size,
setSize,
mutate,
error: !!error,
2022-02-11 17:24:33 +00:00
}
}