omnivore/packages/api/test/resolvers/article.test.ts

2514 lines
69 KiB
TypeScript
Raw Permalink Normal View History

2022-02-24 06:56:00 +00:00
import * as chai from 'chai'
2022-02-11 17:24:33 +00:00
import { expect } from 'chai'
import chaiString from 'chai-string'
2023-03-14 10:26:45 +00:00
import 'mocha'
import sinon from 'sinon'
2023-09-06 11:31:34 +00:00
import { DeepPartial } from 'typeorm'
2023-10-23 04:24:19 +00:00
import { Group } from '../../src/entity/groups/group'
2023-09-06 11:31:34 +00:00
import { Highlight } from '../../src/entity/highlight'
import { Label } from '../../src/entity/label'
import {
DirectionalityType,
LibraryItem,
LibraryItemState,
} from '../../src/entity/library_item'
import { UploadFile } from '../../src/entity/upload_file'
2023-03-14 10:26:45 +00:00
import { User } from '../../src/entity/user'
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
import {
2023-09-06 11:31:34 +00:00
ArticleSavingRequestStatus,
2023-03-14 10:26:45 +00:00
BulkActionType,
HighlightType,
2023-09-19 11:33:43 +00:00
PageType,
2023-03-14 10:26:45 +00:00
SyncUpdatedItemEdge,
UpdateReason,
2024-01-21 06:11:53 +00:00
UploadFileStatus,
2023-03-14 10:26:45 +00:00
} from '../../src/generated/graphql'
2023-08-23 04:15:50 +00:00
import { getRepository } from '../../src/repository'
2023-10-23 04:24:19 +00:00
import { createGroup, deleteGroup } from '../../src/services/groups'
2024-02-21 06:29:21 +00:00
import { createLabel, deleteLabels } from '../../src/services/labels'
2023-09-06 11:31:34 +00:00
import {
2024-08-13 04:08:53 +00:00
countLibraryItems,
createLibraryItems,
2024-02-21 06:29:21 +00:00
createOrUpdateLibraryItem,
CreateOrUpdateLibraryItemArgs,
2023-09-06 11:31:34 +00:00
deleteLibraryItemById,
deleteLibraryItemByUrl,
deleteLibraryItems,
deleteLibraryItemsByUserId,
2023-09-06 11:31:34 +00:00
findLibraryItemById,
findLibraryItemByUrl,
softDeleteLibraryItem,
2024-01-21 06:11:53 +00:00
updateLibraryItem,
2023-09-06 11:31:34 +00:00
} from '../../src/services/library_item'
2023-09-07 04:35:05 +00:00
import { deleteUser } from '../../src/services/user'
import * as createTask from '../../src/utils/createTask'
import * as uploads from '../../src/utils/uploads'
2024-01-31 09:45:49 +00:00
import {
2024-01-31 11:10:57 +00:00
createHighlight,
createTestLibraryItem,
createTestUser,
saveLabelsInLibraryItem,
} from '../db'
2024-02-21 06:29:21 +00:00
import { generateFakeUuid, graphqlRequest, request } from '../util'
chai.use(chaiString)
const archiveLink = async (authToken: string, linkId: string) => {
const query = `
mutation {
setLinkArchived(
input: {
linkId: "${linkId}",
2024-05-16 04:03:41 +00:00
archived: true
}
) {
... on ArchiveLinkSuccess {
linkId
}
... on ArchiveLinkError {
errorCodes
}
}
}
`
2022-02-24 06:56:00 +00:00
return graphqlRequest(query, authToken).expect(200)
}
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
const createArticleQuery = (
url: string,
source: string,
document: string,
title: string
) => {
return `
mutation {
createArticle(input: {
url: "${url}"
source: "${source}"
preparedDocument: {
document: "${document}"
pageInfo: {
contentType: "text/html"
title: "${title}"
}
}
}) {
... on CreateArticleSuccess {
createdArticle {
id
title
content
2022-06-26 10:53:28 +00:00
isArchived
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
}
user {
id
name
}
created
}
... on CreateArticleError {
errorCodes
}
}
}
`
}
const getArticleQuery = (slug: string) => {
return `
query {
article(slug: "${slug}", username: "") {
... on ArticleSuccess {
article {
id
slug
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
content
directionality
highlights {
id
shortId
quote
prefix
suffix
patch
annotation
sharedAt
createdAt
updatedAt
}
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
}
}
... on ArticleError {
errorCodes
}
}
}
`
}
const searchQuery = (keyword = '') => {
return `
query {
search(
after: ""
first: 5
query: "${keyword}") {
... on SearchSuccess {
edges {
cursor
node {
id
url
createdAt
updatedAt
directionality
2022-06-28 02:14:15 +00:00
highlights {
id
}
2023-10-23 04:24:19 +00:00
labels {
id
name
}
recommendations {
name
}
}
}
pageInfo {
hasNextPage
hasPreviousPage
startCursor
endCursor
totalCount
}
}
... on SearchError {
errorCodes
}
}
}
`
}
const savePageQuery = `
mutation SavePage($input: SavePageInput!) {
savePage(input: $input) {
... on SaveSuccess {
url
}
... on SaveError {
errorCodes
}
}
}`
2022-02-11 17:24:33 +00:00
2024-01-29 06:11:07 +00:00
const saveFileQuery = (
clientRequestId: string,
url: string,
uploadFileId: string
) => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
return `
mutation {
saveFile (
input: {
url: "${url}",
source: "test",
2024-01-29 06:11:07 +00:00
clientRequestId: "${clientRequestId}",
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
uploadFileId: "${uploadFileId}",
}
) {
... on SaveSuccess {
url
}
... on SaveError {
errorCodes
}
}
}
`
}
2023-03-06 06:34:50 +00:00
const saveUrlQuery = (
url: string,
state: ArticleSavingRequestStatus | null = null,
labels: string[] | null = null
) => {
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
return `
mutation {
saveUrl(
input: {
url: "${url}",
source: "test",
clientRequestId: "${generateFakeUuid()}",
2024-05-16 04:03:41 +00:00
state: ${state ?? 'null'},
2023-03-06 06:34:50 +00:00
labels: ${
labels
2024-05-16 04:03:41 +00:00
? '[' +
labels.map((label) => `{ name: "${label}" }`).join(',') +
']'
: 'null'
2023-03-06 06:34:50 +00:00
}
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
}
) {
... on SaveSuccess {
url
}
... on SaveError {
errorCodes
}
}
}
`
}
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
const setBookmarkQuery = (articleId: string, bookmark: boolean) => {
return `
mutation {
setBookmarkArticle(
input: {
articleID: "${articleId}",
2024-05-16 04:03:41 +00:00
bookmark: ${String(bookmark)}
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
}
) {
... on SetBookmarkArticleSuccess {
bookmarkedArticle {
id
}
}
... on SetBookmarkArticleError {
errorCodes
}
}
}
`
}
const saveArticleReadingProgressQuery = (
articleId: string,
progress: number,
topPercent: number | null = null,
force: boolean | null = null
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
) => {
return `
mutation {
saveArticleReadingProgress(
input: {
id: "${articleId}",
readingProgressPercent: ${progress},
readingProgressAnchorIndex: 0,
2024-05-16 04:03:41 +00:00
readingProgressTopPercent: ${topPercent ?? 'null'},
force: ${String(force) ?? 'null'}
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
}
) {
... on SaveArticleReadingProgressSuccess {
updatedArticle {
id
readingProgressPercent
2022-06-15 01:42:38 +00:00
readAt
readingProgressTopPercent
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
}
}
... on SaveArticleReadingProgressError {
errorCodes
}
}
}
`
}
2022-07-12 15:07:57 +00:00
const typeaheadSearchQuery = (keyword: string) => {
return `
query {
typeaheadSearch(query: "${keyword}") {
... on TypeaheadSearchSuccess {
items {
id
slug
title
contentReader
2022-07-12 15:07:57 +00:00
}
}
... on TypeaheadSearchError {
errorCodes
}
}
}
`
}
2022-02-11 17:24:33 +00:00
describe('Article API', () => {
let authToken: string
let user: User
2022-02-11 17:24:33 +00:00
before(async () => {
// create test user and login
2022-08-10 08:36:31 +00:00
user = await createTestUser('fakeUser')
2022-02-11 17:24:33 +00:00
const res = await request
.post('/local/debug/fake-user-login')
.send({ fakeEmail: user.email })
2024-05-16 04:03:41 +00:00
authToken = res.body.authToken as string
2022-02-11 17:24:33 +00:00
})
after(async () => {
// clean up
2023-09-07 04:35:05 +00:00
await deleteUser(user.id)
2022-02-11 17:24:33 +00:00
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
describe('CreateArticle', () => {
2022-02-11 17:24:33 +00:00
let query = ''
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
let url = ''
let source = ''
let document = ''
let title = ''
2023-09-06 11:31:34 +00:00
let itemId = ''
2022-02-11 17:24:33 +00:00
2024-05-16 04:03:41 +00:00
beforeEach(() => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
query = createArticleQuery(url, source, document, title)
2022-02-11 17:24:33 +00:00
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
context('when saving from document', () => {
before(() => {
url = 'https://blog.omnivore.app/p/testing-is-fun-with-omnivore'
source = 'puppeteer-parse'
document = '<p>test</p>'
title = 'new title'
})
after(async () => {
2023-09-06 11:31:34 +00:00
await deleteLibraryItemById(itemId, user.id)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
2022-02-24 06:56:00 +00:00
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
it('should create an article', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.createArticle.createdArticle.title).to.eql(title)
2024-05-16 04:03:41 +00:00
itemId = res.body.data.createArticle.createdArticle.id as string
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
})
2022-06-26 10:53:28 +00:00
context('when saving an archived article', () => {
before(async () => {
2022-10-12 10:05:17 +00:00
url = 'https://blog.omnivore.app/saving-archived-article.com'
2022-06-26 10:53:28 +00:00
source = 'puppeteer-parse'
document = '<p>test</p>'
title = 'new title'
2024-02-06 05:10:54 +00:00
const item = await createOrUpdateLibraryItem(
2022-06-26 10:53:28 +00:00
{
2023-09-06 11:31:34 +00:00
readableContent: document,
2022-06-26 10:53:28 +00:00
slug: 'test saving an archived article slug',
title,
2023-09-06 11:31:34 +00:00
user: { id: user.id },
originalUrl: url,
archivedAt: new Date(),
state: LibraryItemState.Archived,
2022-06-26 10:53:28 +00:00
},
user.id,
undefined,
true
2023-09-06 11:31:34 +00:00
)
itemId = item.id
2022-08-05 03:44:02 +00:00
})
after(async () => {
2023-09-06 11:31:34 +00:00
await deleteLibraryItemById(itemId, user.id)
2022-06-26 10:53:28 +00:00
})
2022-10-12 10:05:17 +00:00
it('unarchives the article', async () => {
2022-06-26 10:53:28 +00:00
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.createArticle.createdArticle.isArchived).to.false
})
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
describe('GetArticle', () => {
const realSlug = 'testing-is-really-fun-with-omnivore'
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
let query = ''
let slug = ''
2023-09-06 11:31:34 +00:00
let itemId: string
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
before(async () => {
const itemToCreate: CreateOrUpdateLibraryItemArgs = {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
title: 'test title',
slug: realSlug,
2023-09-06 11:31:34 +00:00
readingProgressTopPercent: 100,
user,
originalUrl: 'https://blog.omnivore.app/test-with-omnivore',
directionality: DirectionalityType.RTL,
2022-08-05 03:44:02 +00:00
}
2024-05-15 07:53:42 +00:00
const item = await createOrUpdateLibraryItem(
itemToCreate,
user.id,
undefined,
true
)
2023-09-06 11:31:34 +00:00
itemId = item.id
2024-02-06 05:10:54 +00:00
// save highlights
await createHighlight(
{
shortId: 'test short id',
patch: 'test patch',
quote: 'test quote',
user,
libraryItem: item,
},
itemId,
user.id
)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
after(async () => {
2023-09-06 11:31:34 +00:00
await deleteLibraryItemById(itemId, user.id)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
2024-05-16 04:03:41 +00:00
beforeEach(() => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
query = getArticleQuery(slug)
})
2023-09-21 09:17:47 +00:00
context('when item exists', () => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
before(() => {
slug = realSlug
})
2023-09-21 09:17:47 +00:00
it('should return the item', async () => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.article.article.slug).to.eql(slug)
expect(res.body.data.article.article.directionality).to.eql(
DirectionalityType.RTL
)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
it('should return highlights', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.article.article.highlights).to.length(1)
})
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
2023-09-21 09:17:47 +00:00
context('when item is failed to process', () => {
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
before(async () => {
2023-09-06 11:31:34 +00:00
await updateLibraryItem(
itemId,
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
{
2023-09-06 11:31:34 +00:00
state: LibraryItemState.Processing,
2022-05-04 09:54:59 +00:00
savedAt: new Date(Date.now() - 1000 * 60),
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
},
2023-09-06 11:31:34 +00:00
user.id
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
)
})
it('should return unable to parse', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.article.article.content).to.eql(
'<p>We were unable to parse this page.</p>'
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
)
})
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
2023-09-21 09:17:47 +00:00
context('query with id instead of slug', () => {
before(() => {
slug = itemId
})
it('returns the item', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.article.article.id).to.eql(slug)
})
})
context('when item does not exist', () => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
before(() => {
slug = 'not-a-real-slug'
})
it('should return an error', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.article.errorCodes).to.eql(['NOT_FOUND'])
})
})
})
describe('SavePage', () => {
2024-05-15 07:53:42 +00:00
const title = 'Example Title'
2022-10-12 10:05:17 +00:00
let url = 'https://blog.omnivore.app'
2024-05-15 07:53:42 +00:00
const originalContent =
'<html dir="rtl"><body><div>Example Content</div></body></html>'
2024-05-15 07:53:42 +00:00
const source = 'puppeteer-parse'
2023-09-21 09:17:47 +00:00
context('when we save a new item', () => {
2022-08-05 03:44:02 +00:00
after(async () => {
2023-09-06 14:53:58 +00:00
await deleteLibraryItemByUrl(url, user.id)
2022-08-05 03:44:02 +00:00
})
it('should return a slugged url', async () => {
const res = await graphqlRequest(savePageQuery, authToken, {
input: {
url,
originalContent,
title,
clientRequestId: generateFakeUuid(),
source,
},
}).expect(200)
2022-02-24 06:56:00 +00:00
expect(res.body.data.savePage.url).to.startsWith(
'http://localhost:3000/fakeUser/example-title-'
)
})
})
2023-09-21 09:17:47 +00:00
context('when we save a item that is already archived', () => {
2022-08-05 03:44:02 +00:00
before(() => {
2022-10-12 10:05:17 +00:00
url = 'https://blog.omnivore.app/new-url'
2022-08-05 03:44:02 +00:00
})
after(async () => {
2023-09-07 04:35:05 +00:00
await deleteLibraryItemByUrl(url, user.id)
2022-08-05 03:44:02 +00:00
})
2023-09-21 09:17:47 +00:00
it('it should return that item in the Search Query', async () => {
const variables = {
input: {
url,
originalContent,
title,
clientRequestId: generateFakeUuid(),
source,
},
}
await graphqlRequest(savePageQuery, authToken, variables).expect(200)
// Save a link, then archive it
2024-01-21 06:11:53 +00:00
let allLinks = await graphqlRequest(
searchQuery('in:inbox'),
authToken
).expect(200)
2024-05-16 04:03:41 +00:00
const justSavedId = allLinks.body.data.search.edges[0].node.id as string
2022-08-05 03:44:02 +00:00
await archiveLink(authToken, justSavedId)
// test the negative case, ensuring the archive link wasn't returned
2024-01-21 06:11:53 +00:00
allLinks = await graphqlRequest(
searchQuery('in:inbox'),
authToken
).expect(200)
2023-09-07 05:17:23 +00:00
expect(allLinks.body.data.search.edges[0]?.node?.url).to.not.eq(url)
// Now save the link again, and ensure it is returned
await graphqlRequest(savePageQuery, authToken, variables).expect(200)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
2024-01-21 06:11:53 +00:00
allLinks = await graphqlRequest(
searchQuery('in:inbox'),
authToken
).expect(200)
expect(allLinks.body.data.search.edges[0].node.id).to.eq(justSavedId)
2023-09-07 05:17:23 +00:00
expect(allLinks.body.data.search.edges[0].node.url).to.eq(url)
expect(allLinks.body.data.search.edges[0].node.directionality).to.eq(
'RTL'
)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
})
context('when we also want to save labels and archives the item', () => {
before(() => {
url = 'https://blog.omnivore.app/new-url-2'
})
after(async () => {
await deleteLibraryItemByUrl(url, user.id)
})
2023-09-21 09:17:47 +00:00
it('saves the labels and archives the item', async () => {
const state = ArticleSavingRequestStatus.Archived
const labels = [{ name: 'test name' }, { name: 'test name 2' }]
await graphqlRequest(savePageQuery, authToken, {
input: {
url,
state,
labels,
originalContent,
clientRequestId: generateFakeUuid(),
source,
},
}).expect(200)
2023-09-06 11:31:34 +00:00
const savedItem = await findLibraryItemByUrl(url, user.id)
expect(savedItem?.archivedAt).to.not.be.null
expect(savedItem?.labels?.map((l) => l.name)).to.include.members(
labels.map((l) => l.name)
)
})
})
2023-10-23 07:47:21 +00:00
context('when the source is rss-feeder and url is from youtube.com', () => {
const source = 'rss-feeder'
2024-07-16 01:57:00 +00:00
const stub = sinon.stub(createTask, 'enqueueFetchContentJob')
sinon.stub(createTask, 'enqueueProcessYouTubeVideo')
before(() => {
2023-10-23 07:47:21 +00:00
url = 'https://www.youtube.com/watch?v=123'
})
after(async () => {
await deleteLibraryItemByUrl(url, user.id)
sinon.restore()
})
it('does not parse in the backend', async () => {
await graphqlRequest(savePageQuery, authToken, {
input: {
url,
source,
originalContent,
clientRequestId: generateFakeUuid(),
},
}).expect(200)
expect(stub).not.to.have.been.called
})
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
describe('SaveUrl', () => {
let query = ''
2024-05-15 07:53:42 +00:00
const url = 'https://blog.omnivore.app/new-url-1'
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
2023-03-06 06:34:50 +00:00
before(() => {
2024-07-16 01:57:00 +00:00
sinon.replace(
createTask,
'enqueueFetchContentJob',
sinon.fake.resolves('')
)
2023-03-06 06:34:50 +00:00
})
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
beforeEach(() => {
query = saveUrlQuery(url)
})
2023-03-06 06:34:50 +00:00
after(() => {
sinon.restore()
})
2022-08-05 03:44:02 +00:00
2023-03-06 06:34:50 +00:00
afterEach(async () => {
2023-09-06 11:31:34 +00:00
await deleteLibraryItemByUrl(url, user.id)
2023-03-06 06:34:50 +00:00
})
context('when we save a new url', () => {
Remove article saving request (#493) * Add state and taskName in elastic page mappings * Add state and taskName in elastic page interface * Create page with PROCESSING state before scrapping * Update createArticleRequest API * Fix tests * Add default state for pages * Update createArticle API * Update save page * Update save file * Update saving item description * Show unable to parse content for failed page * Fix date parsing * Search for not failed pages * Fix tests * Add test for saveUrl * Update get article saving request api * Update get article test * Add test for articleSavingRequest API * Add test for failure * Return new page id if clientRequestId empty * Update clientRequestId in savePage * Update clientRequestId in saveFile * Replace article with slug in articleSavingRequest * Add slug in articleSavingRequest response * Depreciate article * Use slug in web * Remove article and highlight fragments * Query article.slug on Prod * Show unable to parse description for failed page * Fix a bug having duplicate pages when saving the same url multiple times * Add state in response * Rename variables in removeArticle API * Rename state * Add state in response in web * Make state an enum * Open temporary page by link id * Use an empty reader view as the background for loading pages * Progressively load the article page as content is loaded * Add includePending flag in getArticles API * Set includePending = true in web * Add elastic update mappings in migration script * Add elastic mappings in docker image * Move index_settings.json to migrate package * Remove elastic index creation in api * Move elastic migrations to a separate directory * Remove index_settings from api docker image Co-authored-by: Jackson Harper <jacksonh@gmail.com>
2022-04-29 05:41:06 +00:00
it('should return a slugged url', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.saveUrl.url).to.startsWith(
'http://localhost:3000/fakeUser/links/'
)
})
})
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
describe('setBookmarkArticle', () => {
2023-09-06 11:31:34 +00:00
let itemId: string
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
before(async () => {
const itemToSave: CreateOrUpdateLibraryItemArgs = {
2023-09-06 11:31:34 +00:00
user,
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
title: 'test title',
2023-09-06 11:31:34 +00:00
readableContent: '<p>test</p>',
originalUrl: 'https://blog.omnivore.app/setBookmarkArticle',
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
slug: 'test-with-omnivore',
}
const item = await createOrUpdateLibraryItem(
itemToSave,
user.id,
undefined,
true
)
2023-09-06 11:31:34 +00:00
itemId = item.id
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
after(async () => {
2023-09-06 11:31:34 +00:00
await deleteLibraryItemById(itemId, user.id)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
it('soft deletes the item', async () => {
2023-09-07 05:17:23 +00:00
await graphqlRequest(setBookmarkQuery(itemId, false), authToken).expect(
200
)
const item = await findLibraryItemById(itemId, user.id)
expect(item?.state).to.eql(LibraryItemState.Deleted)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
})
describe('saveArticleReadingProgressResolver', () => {
let query = ''
2023-09-06 11:31:34 +00:00
let itemId = ''
2024-05-15 07:53:42 +00:00
const progress = 0.5
const topPercent: number | null = null
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
before(async () => {
2023-09-06 11:31:34 +00:00
itemId = (await createTestLibraryItem(user.id)).id
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
after(async () => {
2023-09-06 11:31:34 +00:00
await deleteLibraryItemById(itemId, user.id)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
it('saves a reading progress on an article', async () => {
2023-09-06 11:31:34 +00:00
query = saveArticleReadingProgressQuery(itemId, progress, topPercent)
const res = await graphqlRequest(query, authToken).expect(200)
expect(
res.body.data.saveArticleReadingProgress.updatedArticle
.readingProgressPercent
).to.eq(progress)
expect(res.body.data.saveArticleReadingProgress.updatedArticle.readAt).not
.null
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
it('should not allow setting the reading progress lower than current progress', async () => {
2023-09-06 11:31:34 +00:00
const firstQuery = saveArticleReadingProgressQuery(itemId, 75)
const firstRes = await graphqlRequest(firstQuery, authToken).expect(200)
expect(
firstRes.body.data.saveArticleReadingProgress.updatedArticle
.readingProgressPercent
).to.eq(75)
// Now try to set to a lower value (50), value should not be updated
2023-09-06 11:31:34 +00:00
const secondQuery = saveArticleReadingProgressQuery(itemId, 50)
const secondRes = await graphqlRequest(secondQuery, authToken).expect(200)
expect(
secondRes.body.data.saveArticleReadingProgress.updatedArticle
.readingProgressPercent
).to.eq(75)
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
it('does not save topPercent if not undefined', async () => {
2023-09-06 11:31:34 +00:00
query = saveArticleReadingProgressQuery(itemId, progress, null)
const res = await graphqlRequest(query, authToken).expect(200)
expect(
res.body.data.saveArticleReadingProgress.updatedArticle
.readingProgressTopPercent
2023-09-07 05:17:23 +00:00
).to.eq(0)
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
it('saves topPercent if defined', async () => {
const topPercent = 0.2
2023-09-06 11:31:34 +00:00
query = saveArticleReadingProgressQuery(itemId, progress, topPercent)
const res = await graphqlRequest(query, authToken).expect(200)
expect(
res.body.data.saveArticleReadingProgress.updatedArticle
.readingProgressTopPercent
).to.eql(topPercent)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
2023-03-10 04:42:57 +00:00
it('saves topPercent as 0 if defined as 0', async () => {
const topPercent = 0
query = saveArticleReadingProgressQuery(
itemId,
progress,
topPercent,
true
)
2023-03-10 04:42:57 +00:00
const res = await graphqlRequest(query, authToken).expect(200)
expect(
res.body.data.saveArticleReadingProgress.updatedArticle
.readingProgressTopPercent
).to.eql(topPercent)
})
it('returns BAD_DATA error if top position is greater than bottom position', async () => {
2023-09-06 11:31:34 +00:00
query = saveArticleReadingProgressQuery(itemId, 0.5, 0.8)
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.saveArticleReadingProgress.errorCodes).to.eql([
'BAD_DATA',
])
})
context('when force is true', () => {
before(async () => {
2024-01-21 06:11:53 +00:00
itemId = (
2024-02-06 05:10:54 +00:00
await createOrUpdateLibraryItem(
2024-01-21 06:11:53 +00:00
{
user: { id: user.id },
originalUrl: 'https://blog.omnivore.app/setBookmarkArticle',
slug: 'test-with-omnivore',
readableContent: '<p>test</p>',
title: 'test title',
readingProgressBottomPercent: 100,
readingProgressTopPercent: 80,
},
user.id,
undefined,
true
2024-01-21 06:11:53 +00:00
)
).id
})
after(async () => {
await deleteLibraryItemById(itemId, user.id)
})
it('ignore position check if force is true', async () => {
query = saveArticleReadingProgressQuery(itemId, 20, 10, true)
const res = await graphqlRequest(query, authToken).expect(200)
expect(
res.body.data.saveArticleReadingProgress.updatedArticle
.readingProgressPercent
).to.eql(20)
expect(
res.body.data.saveArticleReadingProgress.updatedArticle
.readingProgressTopPercent
).to.eql(10)
})
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
describe('SaveFile', () => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
let query = ''
let url = ''
let uploadFileId = ''
2024-01-29 06:11:07 +00:00
let itemId = ''
before(async () => {
2024-02-06 05:10:54 +00:00
const item = await createOrUpdateLibraryItem(
2024-01-29 06:11:07 +00:00
{
user: { id: user.id },
originalUrl: 'https://blog.omnivore.app/setBookmarkArticle',
slug: 'test-with-omnivore',
readableContent: '<p>test</p>',
title: 'test title',
readingProgressBottomPercent: 100,
readingProgressTopPercent: 80,
},
user.id,
undefined,
true
2024-01-29 06:11:07 +00:00
)
itemId = item.id
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
sinon.replace(
uploads,
'getStorageFileDetails',
sinon.fake.resolves({ fileUrl: 'fake url', md5Hash: 'fake hash' })
)
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
beforeEach(() => {
2024-01-29 06:11:07 +00:00
query = saveFileQuery(itemId, url, uploadFileId)
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
})
after(() => {
sinon.restore()
})
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
context('when the file is not uploaded', () => {
2024-05-16 04:03:41 +00:00
before(() => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
url = 'fake url'
uploadFileId = generateFakeUuid()
})
2022-08-05 03:44:02 +00:00
it('should return Unauthorized error', async () => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.saveFile.errorCodes).to.eql(['UNAUTHORIZED'])
})
})
context('when the file is uploaded', () => {
before(async () => {
2022-10-12 10:05:17 +00:00
url = 'https://blog.omnivore.app/'
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
const uploadFile = await getRepository(UploadFile).save({
fileName: 'test.pdf',
contentType: 'application/pdf',
url: url,
user: user,
status: UploadFileStatus.Initialized,
})
uploadFileId = uploadFile.id
})
2022-08-05 03:44:02 +00:00
it('should return the new url', async () => {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.saveFile.url).to.startsWith(
'http://localhost:3000/fakeUser/links'
)
})
})
})
describe('Search API', () => {
const url = 'https://blog.omnivore.app/p/getting-started-with-omnivore'
2023-09-06 11:31:34 +00:00
const items: LibraryItem[] = []
const highlights: Highlight[] = []
2023-06-30 09:28:37 +00:00
const searchedKeyword = 'aaabbbccc'
let query = ''
let keyword = ''
before(async () => {
const readingProgressArray = [0, 2, 97, 98, 100]
// Create some test items
for (let i = 0; i < 5; i++) {
const itemToSave: CreateOrUpdateLibraryItemArgs = {
2023-09-06 11:31:34 +00:00
user,
title: 'test title',
2023-09-06 11:31:34 +00:00
readableContent: `<p>test ${searchedKeyword}</p>`,
slug: 'test slug',
2023-09-06 11:31:34 +00:00
originalUrl: `${url}/${i}`,
2023-03-14 10:26:45 +00:00
siteName: 'Example',
readingProgressBottomPercent: readingProgressArray[i],
}
const item = await createOrUpdateLibraryItem(
itemToSave,
user.id,
undefined,
true
)
2023-09-06 11:31:34 +00:00
items.push(item)
// Create some test highlights
2023-09-06 11:31:34 +00:00
const highlightToSave: DeepPartial<Highlight> = {
patch: 'test patch',
2023-09-07 05:17:23 +00:00
shortId: `test shortId${i}`,
2023-09-06 11:31:34 +00:00
user,
quote: '<p>search highlight</p>',
2023-09-07 14:21:20 +00:00
libraryItem: item,
}
2023-09-07 05:17:23 +00:00
const highlight = await createHighlight(
highlightToSave,
item.id,
user.id
)
highlights.push(highlight)
}
})
2024-05-16 04:03:41 +00:00
beforeEach(() => {
query = searchQuery(keyword)
})
2022-08-05 03:44:02 +00:00
after(async () => {
await deleteLibraryItemsByUserId(user.id)
2022-08-05 03:44:02 +00:00
})
context('when type:highlights is not in the query', () => {
before(() => {
2023-06-30 09:28:37 +00:00
keyword = searchedKeyword
})
it('should return items in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.edges.length).to.eql(5)
2023-09-06 11:31:34 +00:00
expect(res.body.data.search.edges[0].node.id).to.eq(items[4].id)
expect(res.body.data.search.edges[1].node.id).to.eq(items[3].id)
expect(res.body.data.search.edges[2].node.id).to.eq(items[2].id)
expect(res.body.data.search.edges[3].node.id).to.eq(items[1].id)
expect(res.body.data.search.edges[4].node.id).to.eq(items[0].id)
})
2022-06-28 02:14:15 +00:00
it('should return highlights in items', async () => {
2022-06-28 02:14:15 +00:00
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.edges[0].node.highlights.length).to.eql(1)
expect(res.body.data.search.edges[0].node.highlights[0].id).to.eq(
highlights[4].id
)
})
})
context('when is:unread is in the query', () => {
before(() => {
2023-06-30 09:28:37 +00:00
keyword = `'${searchedKeyword}' is:unread`
})
it('returns unread articles in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.edges.length).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when is:reading is in the query', () => {
before(() => {
keyword = `'${searchedKeyword}' is:reading`
})
it('returns reading articles in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.edges.length).to.eq(3)
expect(res.body.data.search.edges[0].node.id).to.eq(items[3].id)
expect(res.body.data.search.edges[1].node.id).to.eq(items[2].id)
expect(res.body.data.search.edges[2].node.id).to.eq(items[1].id)
})
})
context('when is:read is in the query', () => {
before(() => {
keyword = `'${searchedKeyword}' is:read`
})
it('returns fully read articles in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.edges.length).to.eq(1)
2023-09-06 11:31:34 +00:00
expect(res.body.data.search.edges[0].node.id).to.eq(items[4].id)
})
})
2023-02-17 10:03:53 +00:00
context('when no:label is in the query', () => {
2024-05-16 04:03:41 +00:00
before(() => {
2023-06-30 09:28:37 +00:00
keyword = `'${searchedKeyword}' no:label`
2023-02-17 10:03:53 +00:00
})
it('returns non-labeled items in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(5)
})
})
context('when no:highlight is in the query', () => {
2024-05-16 04:03:41 +00:00
before(() => {
2023-06-30 09:28:37 +00:00
keyword = `'${searchedKeyword}' no:highlight`
2023-02-17 10:03:53 +00:00
})
it('returns non-highlighted items in descending order', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(0)
})
})
2023-03-14 10:26:45 +00:00
context('when site:${site_name} is in the query', () => {
2024-05-16 04:03:41 +00:00
before(() => {
2023-06-30 09:28:37 +00:00
keyword = `'${searchedKeyword}' site:example`
2023-03-14 10:26:45 +00:00
})
it('returns items from the site', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(5)
})
})
context("when 'in:archive label:test' is in the query", () => {
let items: LibraryItem[] = []
let label: Label
before(async () => {
keyword = 'in:archive label:test'
// Create some test items
label = await createLabel('test', '', user.id)
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
archivedAt: new Date(),
state: LibraryItemState.Archived,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
archivedAt: new Date(),
state: LibraryItemState.Archived,
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
},
],
user.id
)
await saveLabelsInLibraryItem([label], items[0].id, user.id)
await saveLabelsInLibraryItem([label], items[2].id, user.id)
})
after(async () => {
await deleteLabels({ id: label.id }, user.id)
await deleteLibraryItems(items, user.id)
})
it('returns archived items with label test', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
2023-10-13 10:26:02 +00:00
context('when site is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'site:yes-app.com'
2023-10-13 10:26:02 +00:00
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
state: LibraryItemState.Succeeded,
siteName: 'yes-app.com',
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
state: LibraryItemState.Succeeded,
siteName: 'no-app.com',
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns item with matching site', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
2023-10-13 10:26:02 +00:00
context('when wildcard site is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
2023-10-13 10:26:02 +00:00
keyword = 'site:app.com'
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
state: LibraryItemState.Succeeded,
siteName: 'yes-app.com',
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
state: LibraryItemState.Succeeded,
siteName: 'no-app.com',
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns item with matching search query', async () => {
2023-10-13 10:26:02 +00:00
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(2)
})
})
context('when in:inbox no:subscription label:test is in the query', () => {
let items: LibraryItem[] = []
let label: Label
before(async () => {
keyword = 'in:inbox no:subscription label:test'
// Create some test items
label = await createLabel('test', '', user.id)
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
subscription: 'test subscription',
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
archivedAt: new Date(),
state: LibraryItemState.Archived,
},
],
user.id
)
await saveLabelsInLibraryItem([label], items[0].id, user.id)
await saveLabelsInLibraryItem([label], items[1].id, user.id)
await saveLabelsInLibraryItem([label], items[2].id, user.id)
})
after(async () => {
await deleteLabels({ id: label.id }, user.id)
await deleteLibraryItems(items, user.id)
})
it('returns library items with label test', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
2023-11-23 13:33:04 +00:00
context('when wildcard search for labels', () => {
2024-05-16 04:03:41 +00:00
const items: LibraryItem[] = []
2023-11-23 13:33:04 +00:00
let labelIds: string[]
before(async () => {
keyword = 'label:test/*'
// Create some test items
const label1 = await createLabel('test/one', '', user.id)
const label2 = await createLabel('test/two', '', user.id)
labelIds = [label1.id, label2.id]
2024-05-16 04:03:41 +00:00
const itemsToSave = [
{
user,
title: 'test title wildcard',
readableContent: '<p>test wildcard</p>',
slug: 'test slug wildcard',
originalUrl: `${url}/wildcard`,
},
{
user,
title: 'test title wildcard 1',
readableContent: '<p>test wildcard</p>',
slug: 'test slug wildcard 1',
originalUrl: `${url}/wildcard_1`,
},
{
user,
title: 'test title wildcard 2',
readableContent: '<p>test wildcard</p>',
slug: 'test slug wildcard 2',
originalUrl: `${url}/wildcard_2`,
},
]
for (const item of itemsToSave) {
const savedItem = await createOrUpdateLibraryItem(
item,
user.id,
undefined,
true
)
items.push(savedItem)
}
2023-11-23 13:33:04 +00:00
await saveLabelsInLibraryItem([label1], items[0].id, user.id)
await saveLabelsInLibraryItem([label2], items[1].id, user.id)
})
after(async () => {
await deleteLabels(labelIds, user.id)
await deleteLibraryItems(items, user.id)
})
it('returns library items with label test/one and test/two', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(2)
2024-05-16 04:03:41 +00:00
expect(res.body.data.search.edges[0].node.id).to.eq(items[1].id)
expect(res.body.data.search.edges[1].node.id).to.eq(items[0].id)
2023-11-23 13:33:04 +00:00
})
})
context('when type:file label:test is in the query', () => {
let items: LibraryItem[] = []
let label: Label
before(async () => {
keyword = 'type:file label:test'
// Create some test items
label = await createLabel('test', '', user.id)
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
itemType: PageType.File,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
itemType: PageType.File,
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
},
],
user.id
)
await saveLabelsInLibraryItem([label], items[0].id, user.id)
await saveLabelsInLibraryItem([label], items[2].id, user.id)
})
after(async () => {
await deleteLabels({ id: label.id }, user.id)
await deleteLibraryItems(items, user.id)
})
it('returns files with label test', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when in:archive is:unread is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'in:archive is:unread'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
itemType: PageType.File,
archivedAt: new Date(),
2024-02-28 09:24:02 +00:00
state: LibraryItemState.Archived,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
archivedAt: new Date(),
2024-02-28 09:24:02 +00:00
state: LibraryItemState.Archived,
readingProgressBottomPercent: 100,
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns unread archived items', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when rss:feed in:archive is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'rss:feed in:archive'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
subscription: 'feed',
archivedAt: new Date(),
2024-02-28 09:24:02 +00:00
state: LibraryItemState.Archived,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
subscription: 'feed',
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
archivedAt: new Date(),
2024-02-28 09:24:02 +00:00
state: LibraryItemState.Archived,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns archived feed items', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when in:trash is:unread is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'in:trash is:unread'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
deletedAt: new Date(),
2024-02-28 09:24:02 +00:00
state: LibraryItemState.Deleted,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
readingProgressBottomPercent: 100,
deletedAt: new Date(),
2024-02-28 09:24:02 +00:00
state: LibraryItemState.Deleted,
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns unfinished deleted items', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
2023-10-10 04:52:49 +00:00
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when readPosition:>20 readPosition:<50 is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'readPosition:>20 readPosition:<50'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
readingProgressBottomPercent: 40,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
readingProgressBottomPercent: 10,
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
readingProgressBottomPercent: 100,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns items with reading progress between 20% and 50% exclusively', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
2023-10-13 10:26:02 +00:00
context(
'when wordsCount:>=10000 wordsCount:<=20000 is in the query',
() => {
let items: LibraryItem[] = []
2023-10-10 04:52:49 +00:00
2023-10-13 10:26:02 +00:00
before(async () => {
keyword = 'wordsCount:>=10000 wordsCount:<=20000'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
wordCount: 10000,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
wordCount: 8000,
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
wordCount: 100000,
},
],
user.id
)
})
2023-10-10 04:52:49 +00:00
2023-10-13 10:26:02 +00:00
after(async () => {
await deleteLibraryItems(items, user.id)
})
2023-10-10 04:52:49 +00:00
2023-10-13 10:26:02 +00:00
it('returns items with words count between 10000 and 20000 inclusively', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
2023-10-13 10:26:02 +00:00
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
}
)
2023-10-23 04:24:19 +00:00
context('when recommendedBy:* is in the query', () => {
let items: LibraryItem[] = []
let group: Group
before(async () => {
keyword = 'recommendedBy:*'
group = (
await createGroup({
admin: user,
name: 'test group',
})
)[0]
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
recommendations: [
{
recommender: user,
group,
},
],
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
await deleteGroup(group.id)
})
it('returns recommended items', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
expect(
res.body.data.search.edges[0].node.recommendations[0].name
).to.eq(group.name)
})
})
context('when site:youtube.com is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'site:youtube.com'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
2023-10-26 14:15:33 +00:00
originalUrl: 'https://www.youtube.com/watch?v=Omnivore',
itemType: PageType.Video,
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns youtube videos', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
2023-10-26 14:15:33 +00:00
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
context('when site:wikipedia is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'site:wikipedia'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: 'https://en.wikipedia.org/wiki/Omnivore',
},
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns wikipedia pages', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(1)
expect(res.body.data.search.edges[0].node.id).to.eq(items[0].id)
})
})
2023-12-06 14:23:32 +00:00
context('when label:test1,test2 is in the query', () => {
let items: LibraryItem[] = []
let label1: Label
let label2: Label
before(async () => {
keyword = 'label:test1,test2'
// Create some test items
label1 = await createLabel('test1', '', user.id)
label2 = await createLabel('test2', '', user.id)
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
2024-01-10 10:07:58 +00:00
savedAt: new Date(1703880588),
2023-12-06 14:23:32 +00:00
},
2023-12-08 09:05:24 +00:00
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
2024-01-10 10:07:58 +00:00
savedAt: new Date(1704880589),
2023-12-08 09:05:24 +00:00
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
2024-01-10 10:07:58 +00:00
savedAt: new Date(1705880590),
2023-12-08 09:05:24 +00:00
},
],
user.id
)
await saveLabelsInLibraryItem([label1], items[0].id, user.id)
await saveLabelsInLibraryItem([label2], items[1].id, user.id)
})
after(async () => {
await deleteLabels({ id: label1.id }, user.id)
await deleteLabels({ id: label2.id }, user.id)
await deleteLibraryItems(items, user.id)
})
it('returns items with label test1 or test2', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(2)
expect(res.body.data.search.edges[0].node.id).to.eq(items[1].id)
expect(res.body.data.search.edges[1].node.id).to.eq(items[0].id)
})
})
context('when label:test1 OR label:test2 is in the query', () => {
let items: LibraryItem[] = []
let label1: Label
let label2: Label
before(async () => {
keyword = 'label:test1 OR label:test2'
// Create some test items
label1 = await createLabel('test1', '', user.id)
label2 = await createLabel('test2', '', user.id)
items = await createLibraryItems(
[
{
user,
title: 'test title 1',
readableContent: '<p>test 1</p>',
slug: 'test slug 1',
originalUrl: `${url}/test1`,
2024-01-10 10:07:58 +00:00
savedAt: new Date(1703880588),
2023-12-08 09:05:24 +00:00
},
2023-12-06 14:23:32 +00:00
{
user,
title: 'test title 2',
readableContent: '<p>test 2</p>',
slug: 'test slug 2',
originalUrl: `${url}/test2`,
2024-01-10 10:07:58 +00:00
savedAt: new Date(1704880589),
2023-12-06 14:23:32 +00:00
},
{
user,
title: 'test title 3',
readableContent: '<p>test 3</p>',
slug: 'test slug 3',
originalUrl: `${url}/test3`,
2024-01-10 10:07:58 +00:00
savedAt: new Date(1705880590),
2023-12-06 14:23:32 +00:00
},
],
user.id
)
await saveLabelsInLibraryItem([label1], items[0].id, user.id)
await saveLabelsInLibraryItem([label2], items[1].id, user.id)
})
after(async () => {
await deleteLabels({ id: label1.id }, user.id)
await deleteLabels({ id: label2.id }, user.id)
await deleteLibraryItems(items, user.id)
})
it('returns items with label test1 or test2', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eq(2)
expect(res.body.data.search.edges[0].node.id).to.eq(items[1].id)
expect(res.body.data.search.edges[1].node.id).to.eq(items[0].id)
})
})
context('when sort:score is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'sort:score score'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'score',
slug: 'test 1',
originalUrl: `${url}/test1`,
},
{
user,
title: 'score score',
slug: 'test 2',
originalUrl: `${url}/test2`,
},
{
user,
title: 'score score score',
slug: 'test 3',
originalUrl: `${url}/test3`,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns items in descending order of score', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eql(3)
expect(res.body.data.search.edges[0].node.id).to.eq(items[2].id)
2023-12-18 02:40:10 +00:00
expect(res.body.data.search.edges[1].node.id).to.eq(items[1].id)
expect(res.body.data.search.edges[2].node.id).to.eq(items[0].id)
})
})
context('when sort:wordscount is in the query', () => {
let items: LibraryItem[] = []
before(async () => {
keyword = 'wordscount:>=10000 sort:wordscount'
// Create some test items
items = await createLibraryItems(
[
{
user,
title: 'test 1',
slug: 'test 1',
originalUrl: `${url}/test1`,
wordCount: 10000,
},
{
user,
title: 'test 2',
slug: 'test 2',
originalUrl: `${url}/test2`,
wordCount: 20000,
},
{
user,
title: 'test 3',
slug: 'test 3',
originalUrl: `${url}/test3`,
wordCount: 30000,
},
],
user.id
)
})
after(async () => {
await deleteLibraryItems(items, user.id)
})
it('returns items in descending order of word count', async () => {
const res = await graphqlRequest(query, authToken).expect(200)
expect(res.body.data.search.pageInfo.totalCount).to.eql(3)
expect(res.body.data.search.edges[0].node.id).to.eq(items[2].id)
expect(res.body.data.search.edges[1].node.id).to.eq(items[1].id)
expect(res.body.data.search.edges[2].node.id).to.eq(items[0].id)
})
})
})
2022-07-12 15:07:57 +00:00
describe('TypeaheadSearch API', () => {
2023-09-06 11:31:34 +00:00
const items: LibraryItem[] = []
2022-07-12 15:07:57 +00:00
let query = ''
2024-05-15 07:53:42 +00:00
const keyword = 'typeahead'
2022-07-12 15:07:57 +00:00
before(async () => {
// Create some test items
2022-07-12 15:07:57 +00:00
for (let i = 0; i < 5; i++) {
const itemToSave: CreateOrUpdateLibraryItemArgs = {
2023-09-06 11:31:34 +00:00
user,
2023-09-21 09:17:47 +00:00
title: 'typeahead search item',
2023-09-06 11:31:34 +00:00
readableContent: '<p>test</p>',
2022-07-12 15:07:57 +00:00
slug: '',
2023-09-06 11:31:34 +00:00
originalUrl: `https://blog.omnivore.app/p/typeahead-search-${i}`,
2022-07-12 15:07:57 +00:00
}
const item = await createOrUpdateLibraryItem(
itemToSave,
user.id,
undefined,
true
)
2023-09-06 11:31:34 +00:00
items.push(item)
2022-07-12 15:07:57 +00:00
}
})
2024-05-16 04:03:41 +00:00
beforeEach(() => {
2022-07-12 15:07:57 +00:00
query = typeaheadSearchQuery(keyword)
})
2022-08-05 03:44:02 +00:00
after(async () => {
await deleteLibraryItemsByUserId(user.id)
2022-08-05 03:44:02 +00:00
})
it('returns items with typeahead prefix', async () => {
2022-07-12 15:07:57 +00:00
const res = await graphqlRequest(query, authToken).expect(200)
2022-07-14 10:25:20 +00:00
expect(res.body.data.typeaheadSearch.items.length).to.eql(5)
expect(res.body.data.typeaheadSearch.items[0].id).to.eq(items[4].id)
expect(res.body.data.typeaheadSearch.items[1].id).to.eq(items[3].id)
2023-09-06 11:31:34 +00:00
expect(res.body.data.typeaheadSearch.items[2].id).to.eq(items[2].id)
expect(res.body.data.typeaheadSearch.items[3].id).to.eq(items[1].id)
expect(res.body.data.typeaheadSearch.items[4].id).to.eq(items[0].id)
2022-07-12 15:07:57 +00:00
})
})
describe('UpdatesSince API', () => {
const updatesSinceQuery = (since: string) => `
query {
updatesSince(
since: "${since}") {
... on UpdatesSinceSuccess {
edges {
cursor
node {
id
createdAt
updatedAt
pageType
highlights {
id
type
}
}
itemID
updateReason
}
pageInfo {
hasNextPage
hasPreviousPage
startCursor
endCursor
totalCount
}
}
... on UpdatesSinceError {
errorCodes
}
}
}
`
let since: string
2024-05-15 07:53:42 +00:00
const items: LibraryItem[] = []
const deletedItems: LibraryItem[] = []
before(async () => {
// Create some test items
for (let i = 0; i < 5; i++) {
const itemToSave: CreateOrUpdateLibraryItemArgs = {
2023-09-21 09:17:47 +00:00
title: 'test item',
slug: '',
2023-09-06 11:31:34 +00:00
readableContent: '<p>test</p>',
originalUrl: `https://blog.omnivore.app/p/updates-since-${i}`,
2023-09-07 05:17:23 +00:00
user,
}
const item = await createOrUpdateLibraryItem(
itemToSave,
user.id,
undefined,
true
)
2023-09-06 11:31:34 +00:00
items.push(item)
}
2022-07-29 13:22:55 +00:00
// set the since to be the timestamp before deletion
2023-09-07 05:17:23 +00:00
since = items[4].updatedAt.toISOString()
// Delete some items
for (let i = 0; i < 3; i++) {
await softDeleteLibraryItem(items[i].id, user.id)
2023-09-06 11:31:34 +00:00
deletedItems.push(items[i])
}
})
after(async () => {
// Delete all items
await deleteLibraryItemsByUserId(user.id)
})
it('returns items deleted after since', async () => {
const res = await graphqlRequest(
updatesSinceQuery(since),
authToken
).expect(200)
2022-09-30 05:10:58 +00:00
expect(
2024-05-16 04:03:41 +00:00
(res.body.data.updatesSince.edges as SyncUpdatedItemEdge[]).filter(
2022-09-30 05:10:58 +00:00
(e: SyncUpdatedItemEdge) => e.updateReason === UpdateReason.Deleted
).length
).to.eql(3)
expect(res.body.data.updatesSince.edges[0].itemID).to.eq(
2023-09-06 11:31:34 +00:00
deletedItems[2].id
)
expect(res.body.data.updatesSince.edges[1].itemID).to.eq(
2023-09-06 11:31:34 +00:00
deletedItems[1].id
)
expect(res.body.data.updatesSince.edges[2].itemID).to.eq(
2023-09-06 11:31:34 +00:00
deletedItems[0].id
)
expect(res.body.data.updatesSince.edges[0].updateReason).to.eq(
UpdateReason.Deleted
)
})
2024-01-21 06:11:53 +00:00
context(
'when since is -1000000000-01-01T00:00:00Z from android app',
() => {
before(() => {
since = '-1000000000-01-01T00:00:00Z'
})
2024-01-21 06:11:53 +00:00
it('returns all', async () => {
const res = await graphqlRequest(
updatesSinceQuery(since),
authToken
).expect(200)
2024-01-21 06:11:53 +00:00
expect(res.body.data.updatesSince.edges.length).to.eql(5)
})
}
)
context('returns highlights', () => {
let highlight: Highlight
before(async () => {
highlight = await createHighlight(
{
user,
shortId: 'test short id',
quote: 'test',
libraryItem: items[0],
},
items[0].id,
user.id
)
})
it('returns highlights', async () => {
const res = await graphqlRequest(
updatesSinceQuery(since),
authToken
).expect(200)
expect(res.body.data.updatesSince.edges[0].node.highlights[0].id).to.eq(
highlight.id
)
2024-01-21 06:11:53 +00:00
expect(
res.body.data.updatesSince.edges[0].node.highlights[0].type
).to.eq(HighlightType.Highlight)
})
})
})
2023-02-01 10:05:16 +00:00
describe('BulkAction API', () => {
2023-06-08 05:13:37 +00:00
const bulkActionQuery = (action: BulkActionType, query = 'in:all') => `
2023-02-01 10:05:16 +00:00
mutation {
2023-06-08 05:13:37 +00:00
bulkAction (action: ${action}, query: "${query}") {
... on BulkActionSuccess {
2023-02-01 10:05:16 +00:00
success
}
... on BulkActionError {
2023-02-01 10:05:16 +00:00
errorCodes
}
}
}
`
2024-02-02 06:14:41 +00:00
xcontext('when action is MarkAsRead and query is in:unread', () => {
before(async () => {
// Create some test items
for (let i = 0; i < 5; i++) {
2024-02-06 05:10:54 +00:00
await createOrUpdateLibraryItem(
{
user,
itemType: i == 0 ? PageType.Article : PageType.File,
title: 'test item',
readableContent: '<p>test</p>',
slug: '',
state:
i == 0 ? LibraryItemState.Failed : LibraryItemState.Succeeded,
originalUrl: `https://blog.omnivore.app/p/bulk-action-${i}`,
},
user.id,
undefined,
true
)
}
})
2023-02-01 10:05:16 +00:00
after(async () => {
// Delete all items
await deleteLibraryItemsByUserId(user.id)
})
2023-02-01 10:05:16 +00:00
2023-10-13 05:45:09 +00:00
it('marks unread items as read', async () => {
const res = await graphqlRequest(
bulkActionQuery(BulkActionType.MarkAsRead, 'is:unread'),
authToken
).expect(200)
expect(res.body.data.bulkAction.success).to.be.true
const items = await graphqlRequest(
searchQuery('is:unread'),
authToken
).expect(200)
expect(items.body.data.search.pageInfo.totalCount).to.eql(0)
})
})
2023-10-23 13:43:01 +00:00
context(
'when action is Archive and query is published:*..2023-10-01',
() => {
let items: LibraryItem[] = []
2023-06-08 10:52:47 +00:00
2023-10-23 13:43:01 +00:00
before(async () => {
items = await createLibraryItems(
[
{
user,
title: 'test item',
readableContent: '<p>test</p>',
slug: 'test-item',
originalUrl: `https://blog.omnivore.app/p/bulk-action-archive`,
publishedAt: new Date('2023-10-01'),
},
{
user,
title: 'test item 2',
readableContent: '<p>test</p>',
slug: 'test-item-2',
originalUrl: `https://blog.omnivore.app/p/bulk-action-archive-2`,
publishedAt: new Date('2023-10-02'),
},
],
user.id
)
})
after(async () => {
// Delete all items
await deleteLibraryItems(items, user.id)
})
it('archives old items', async () => {
const res = await graphqlRequest(
bulkActionQuery(BulkActionType.Archive, 'published:*..2023-10-01'),
authToken
).expect(200)
expect(res.body.data.bulkAction.success).to.be.true
const response = await graphqlRequest(
searchQuery('in:archive'),
authToken
).expect(200)
expect(response.body.data.search.pageInfo.totalCount).to.eql(1)
expect(response.body.data.search.edges[0].node.id).to.eql(items[0].id)
})
}
)
2023-12-29 10:42:07 +00:00
context('when action is Delete and query contains item id', () => {
2024-05-15 07:53:42 +00:00
const items: LibraryItem[] = []
2023-12-29 10:42:07 +00:00
before(async () => {
// Create some test items
for (let i = 0; i < 5; i++) {
2024-02-06 05:10:54 +00:00
const item = await createOrUpdateLibraryItem(
{
user,
itemType: i == 0 ? PageType.Article : PageType.File,
title: 'test item',
readableContent: '<p>test</p>',
slug: '',
state:
i == 0 ? LibraryItemState.Failed : LibraryItemState.Succeeded,
originalUrl: `https://blog.omnivore.app/p/bulk-action-${i}`,
},
user.id,
undefined,
true
)
2023-12-29 10:42:07 +00:00
items.push(item)
}
})
after(async () => {
// Delete all items
await deleteLibraryItemsByUserId(user.id)
})
it('deletes all items', async () => {
2023-12-29 10:42:07 +00:00
const query = `includes:${items.map((i) => i.id).join(',')}`
const res = await graphqlRequest(
2023-12-29 10:42:07 +00:00
bulkActionQuery(BulkActionType.Delete, query),
authToken
).expect(200)
expect(res.body.data.bulkAction.success).to.be.true
2023-06-08 10:52:47 +00:00
2023-12-29 10:42:07 +00:00
const response = await graphqlRequest(
searchQuery(query),
2023-06-08 10:52:47 +00:00
authToken
).expect(200)
2023-12-29 10:42:07 +00:00
expect(response.body.data.search.pageInfo.totalCount).to.eql(0)
})
2023-02-01 10:05:16 +00:00
})
2024-05-29 09:07:19 +00:00
context(
'when action is MarkAsSeen and query contains a list of item id',
() => {
const items: LibraryItem[] = []
before(async () => {
// Create some test items
for (let i = 0; i < 5; i++) {
const item = await createOrUpdateLibraryItem(
{
user,
title: 'test item',
slug: '',
originalUrl: `https://blog.omnivore.app/p/bulk-action-${i}`,
},
user.id,
undefined,
true
2024-05-29 09:07:19 +00:00
)
items.push(item)
}
})
after(async () => {
// Delete all items
await deleteLibraryItemsByUserId(user.id)
})
it('marks items as seen', async () => {
const query = `includes:${items.map((i) => i.id).join(',')}`
const res = await graphqlRequest(
bulkActionQuery(BulkActionType.MarkAsSeen, query),
authToken
).expect(200)
expect(res.body.data.bulkAction.success).to.be.true
const response = await graphqlRequest(
searchQuery('is:seen'),
authToken
).expect(200)
expect(response.body.data.search.pageInfo.totalCount).to.eql(5)
})
}
)
2023-02-01 10:05:16 +00:00
})
2023-06-09 10:58:41 +00:00
describe('SetFavoriteArticle API', () => {
const setFavoriteArticleQuery = (articleId: string) => `
mutation {
setFavoriteArticle(id: "${articleId}") {
... on SetFavoriteArticleSuccess {
2023-09-07 04:35:05 +00:00
success
2023-06-09 10:58:41 +00:00
}
... on SetFavoriteArticleError {
errorCodes
}
}
}`
let articleId = ''
before(async () => {
const itemToSave: CreateOrUpdateLibraryItemArgs = {
2023-09-06 11:31:34 +00:00
user,
2023-06-09 10:58:41 +00:00
title: 'test setFavoriteArticle',
slug: '',
2023-09-06 11:31:34 +00:00
readableContent: '<p>test</p>',
originalUrl: `https://blog.omnivore.app/p/setFavoriteArticle`,
2023-06-09 10:58:41 +00:00
}
const item = await createOrUpdateLibraryItem(
itemToSave,
user.id,
undefined,
true
)
2023-09-06 11:31:34 +00:00
articleId = item.id
2023-06-09 10:58:41 +00:00
})
after(async () => {
2023-09-21 09:17:47 +00:00
// Delete the item
2023-09-06 11:31:34 +00:00
await deleteLibraryItemById(articleId, user.id)
2023-06-09 10:58:41 +00:00
})
it('favorites the article', async () => {
2023-09-07 04:35:05 +00:00
await graphqlRequest(
2023-06-09 10:58:41 +00:00
setFavoriteArticleQuery(articleId),
authToken
).expect(200)
2024-05-10 08:40:55 +00:00
const item = await findLibraryItemById(articleId, user.id, {
relations: {
labels: true,
},
})
2023-09-06 11:31:34 +00:00
expect(item?.labels?.map((l) => l.name)).to.eql(['Favorites'])
2023-06-09 10:58:41 +00:00
})
})
2024-01-03 04:40:46 +00:00
describe('EmptyTrash API', () => {
const emptyTrashQuery = () => `
mutation {
emptyTrash {
... on EmptyTrashSuccess {
success
}
... on EmptyTrashError {
errorCodes
}
}
}`
2024-05-15 07:53:42 +00:00
const items: LibraryItem[] = []
2024-01-03 04:40:46 +00:00
before(async () => {
// Create some test items
for (let i = 0; i < 5; i++) {
const itemToSave: CreateOrUpdateLibraryItemArgs = {
2024-01-03 04:40:46 +00:00
user,
title: 'test item',
readableContent: '<p>test</p>',
slug: '',
originalUrl: `https://blog.omnivore.app/p/empty-trash-${i}`,
deletedAt: new Date(),
state: LibraryItemState.Deleted,
}
const item = await createOrUpdateLibraryItem(
itemToSave,
user.id,
undefined,
true
)
2024-01-03 04:40:46 +00:00
items.push(item)
}
})
after(async () => {
// Delete all items
await deleteLibraryItemsByUserId(user.id)
})
it('empties the trash', async () => {
await graphqlRequest(emptyTrashQuery(), authToken).expect(200)
2024-08-13 04:08:53 +00:00
const count = await countLibraryItems(
{
query: 'in:trash',
includeDeleted: true,
},
user.id
)
expect(count).to.eql(0)
2024-01-03 04:40:46 +00:00
})
})
2022-02-11 17:24:33 +00:00
})