omnivore/packages/api/package.json

171 lines
5.4 KiB
JSON
Raw Normal View History

2022-02-11 17:24:33 +00:00
{
"name": "@omnivore/api",
"version": "1.0.0",
"license": "UNLICENSED",
"scripts": {
2024-03-23 05:56:56 +00:00
"build": "tsc",
2024-03-24 02:42:29 +00:00
"dev": "ts-node-dev --respawn --transpile-only src/server.ts",
"dev_qp": "ts-node-dev --respawn --transpile-only src/queue-processor.ts",
2024-03-23 07:52:52 +00:00
"start": "node dist/src/server.js",
"start_queue_processor": "node dist/src/queue-processor.js",
2022-02-11 17:24:33 +00:00
"lint": "eslint src --ext ts,js,tsx,jsx",
"lint:fix": "eslint src --fix --ext ts,js,tsx,jsx",
"test:typecheck": "tsc --noEmit",
2024-03-23 07:52:52 +00:00
"test": "nyc mocha -r ts-node/register --config mocha-config.json"
2022-02-11 17:24:33 +00:00
},
"dependencies": {
2024-02-10 08:05:35 +00:00
"@bmatei/apollo-prometheus-exporter": "^3.0.0",
2023-08-11 05:55:29 +00:00
"@google-cloud/logging-winston": "^6.0.0",
"@google-cloud/monitoring": "^4.0.0",
"@google-cloud/opentelemetry-cloud-trace-exporter": "^2.0.0",
2023-08-11 05:55:29 +00:00
"@google-cloud/pubsub": "^4.0.0",
"@google-cloud/storage": "^7.0.1",
"@google-cloud/tasks": "^4.0.0",
2022-11-17 10:40:56 +00:00
"@graphql-tools/utils": "^9.1.1",
2024-02-27 10:49:21 +00:00
"@langchain/openai": "^0.0.14",
2024-03-05 04:43:18 +00:00
"@notionhq/client": "^2.2.14",
"@omnivore/content-handler": "1.0.0",
2023-12-07 10:47:25 +00:00
"@omnivore/liqe": "1.0.0",
2022-02-11 17:24:33 +00:00
"@omnivore/readability": "1.0.0",
2022-09-01 09:09:48 +00:00
"@omnivore/text-to-speech-handler": "1.0.0",
2022-02-11 17:24:33 +00:00
"@opentelemetry/api": "^1.0.1",
"@opentelemetry/core": "^1.3.1",
"@opentelemetry/exporter-jaeger": "^1.0.1",
"@opentelemetry/instrumentation-dns": "^0.31.2",
"@opentelemetry/instrumentation-express": "^0.28.0",
"@opentelemetry/instrumentation-graphql": "^0.29.0",
"@opentelemetry/instrumentation-grpc": "^0.37.0",
"@opentelemetry/instrumentation-http": "^0.27.0",
"@opentelemetry/instrumentation-pg": "^0.35.1",
"@opentelemetry/node": "^0.24.0",
"@opentelemetry/resources": "^1.17.0",
"@opentelemetry/semantic-conventions": "^1.0.1",
2022-02-22 17:43:21 +00:00
"@opentelemetry/tracing": "^0.24.0",
2022-02-11 17:24:33 +00:00
"@sendgrid/mail": "^7.6.0",
"@sentry/integrations": "^7.10.0",
"@sentry/node": "^5.26.0",
"@sentry/tracing": "^7.9.0",
2024-03-13 09:50:07 +00:00
"@types/showdown": "^2.0.6",
2022-07-28 02:44:44 +00:00
"addressparser": "^1.0.1",
"apollo-datasource": "^3.3.1",
"apollo-server-express": "^3.6.3",
"axios": "^0.27.2",
2022-02-11 17:24:33 +00:00
"bcryptjs": "^2.4.3",
2024-01-11 13:03:59 +00:00
"bullmq": "^5.1.1",
"cookie": "^0.5.0",
2022-02-11 17:24:33 +00:00
"cookie-parser": "^1.4.5",
"cors": "^2.8.5",
2023-06-14 04:50:06 +00:00
"csv-stringify": "^6.4.0",
2022-02-11 17:24:33 +00:00
"dataloader": "^2.0.0",
"diff-match-patch": "^1.0.5",
2022-02-11 17:24:33 +00:00
"dompurify": "^2.0.17",
"dot-case": "^3.0.4",
"dotenv": "^8.2.0",
"express": "^4.17.1",
"express-http-context2": "^1.0.0",
2024-02-10 08:05:35 +00:00
"express-prom-bundle": "^7.0.0",
"express-rate-limit": "^6.3.0",
2023-07-28 08:52:28 +00:00
"fast-safe-stringify": "^2.1.1",
"firebase-admin": "^11.5.0",
2023-08-17 09:59:24 +00:00
"googleapis": "^125.0.0",
2022-02-11 17:24:33 +00:00
"graphql": "^15.3.0",
"graphql-fields": "^2.0.3",
2022-02-11 17:24:33 +00:00
"graphql-middleware": "^6.0.10",
"graphql-shield": "^7.5.0",
"highlightjs": "^9.16.2",
"html-entities": "^2.3.2",
2024-01-25 03:18:18 +00:00
"image-size": "^1.0.2",
"intercom-client": "^3.1.4",
"ioredis": "^5.3.2",
2022-02-11 17:24:33 +00:00
"jsonwebtoken": "^8.5.1",
"jwks-rsa": "^2.0.3",
2024-02-27 01:42:03 +00:00
"langchain": "^0.1.21",
"linkedom": "^0.14.9",
"lodash": "^4.17.21",
"luxon": "^3.2.1",
2022-02-11 17:24:33 +00:00
"nanoid": "^3.1.25",
2023-01-05 04:12:48 +00:00
"node-html-markdown": "^1.3.0",
2024-03-31 11:04:55 +00:00
"node-mailjet": "^6.0.5",
Pull ul list out of newsletter blurb next/react doesnt want child elements of the paragraphs Improve formatting Improve wording Use buttons in the subscribe directly blocks Simplify docs on setting up forwarding rules Add extra padding on bottom of help docs Remove unused style Add emails help page Improve formatting Prefetch page content on iOS Reduce the reader overly length now that items are precached Add invalidation when highlights are added to items fix missing index_settings.json file in api dockerfile for creating elastic index (#363) Handle full email address objects in the to param from sendgrid These come in a format like: "jacksonh-dfdf@inbox.omnivore.app" <jacksonh-dfdf@inbox-demo.omnivore.app> New IDs for short highlights because they dont cascade delete now Testing CI issues Simplify test CI test Use promises for async tests Temporarily remove test to debug CI Re-enable re-enable test, return error Specify a userId when looking up saved email pages create a unique url for newsletters without a URL Use 500ms on page test timeouts Increase timeout Dont use deep equal to match newsletter label Run just the labels API Run against just the newsletter emails Run without the page tests Fix Set the allow uncaught flag Remove highlight tests Remove newsletters tests more resolver tests Remove newsetter tests Comment out resolver tests Use nock for external requests in tests Specify puppeteer url for tests Comment out more tests uncomment tests re-enable re-enable email test Re-disable Re-enable one pdf attachment test Re-disable pdf attachment test Use promises on setTimeout tests rm label tests mv label tests into a context Comment out pdf tests Comment out pdf tests Async test Async wrappers Delay when creating test pages More debugging Unique short ids Remove potentially problematic test Fetch page before returning for test handler in before block more debugging More debugging Move errors checks into contexts Use a context when saving newsletters to force index refresh Prettier fix Fix newsletter label check, remove setTimeout Re-enable test timeout on pdf router handler Fix method call comment out PDF test Unique fake username Comment out PDF test Debugging signed urls Re-enable New email pdf test PDF tests Comment out pdf test Add nock stubs for email URLs Use full address for PDF test Remove debug Use full email addresses
2022-03-31 18:21:41 +00:00
"nodemailer": "^6.7.3",
"normalize-url": "^6.1.0",
"oauth": "^0.10.0",
2022-02-11 17:24:33 +00:00
"pg": "^8.3.3",
"postgrator": "^4.2.0",
2024-02-26 08:47:04 +00:00
"posthog-node": "^3.6.3",
2022-02-11 17:24:33 +00:00
"private-ip": "^2.3.3",
2024-02-10 08:05:35 +00:00
"prom-client": "^15.1.0",
"rate-limit-redis": "^4.2.0",
2024-03-30 02:42:01 +00:00
"redis": "^4.6.13",
"rss-parser": "^3.13.0",
2022-02-11 17:24:33 +00:00
"sanitize-html": "^2.3.2",
"sax": "^1.3.0",
2022-02-11 17:24:33 +00:00
"search-query-parser": "^1.6.0",
2024-03-12 13:36:50 +00:00
"showdown": "^2.1.0",
"snake-case": "^4.0.0",
2022-02-11 17:24:33 +00:00
"supertest": "^6.2.2",
"ts-loader": "^9.3.0",
2024-03-12 03:24:52 +00:00
"typeorm": "^0.3.4",
"typeorm-naming-strategies": "^4.1.0",
"underscore": "^1.13.6",
2024-03-30 02:42:01 +00:00
"url-pattern": "^1.0.3",
2022-02-11 17:24:33 +00:00
"urlsafe-base64": "^1.0.0",
"uuid": "^8.3.1",
"voca": "^1.4.0",
2022-09-06 02:19:58 +00:00
"winston": "^3.3.3",
2024-03-12 13:36:50 +00:00
"word-counting": "^1.1.4",
"youtubei": "1.4.0"
2022-02-11 17:24:33 +00:00
},
"devDependencies": {
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
"@istanbuljs/nyc-config-typescript": "^1.0.2",
2022-07-28 03:37:14 +00:00
"@types/addressparser": "^1.0.1",
"@types/analytics-node": "^3.1.7",
"@types/bcryptjs": "^2.4.2",
"@types/chai": "^4.2.18",
2022-06-01 02:30:23 +00:00
"@types/chai-as-promised": "^7.1.5",
"@types/chai-string": "^1.4.2",
"@types/cookie": "^0.4.0",
"@types/cookie-parser": "^1.4.2",
2023-06-14 04:50:06 +00:00
"@types/csv-stringify": "^3.1.0",
"@types/diff-match-patch": "^1.0.32",
"@types/dompurify": "^2.0.4",
"@types/express": "^4.17.7",
"@types/graphql-fields": "^1.3.4",
"@types/highlightjs": "^9.12.2",
"@types/intercom-client": "^2.11.8",
"@types/jsonwebtoken": "^8.5.0",
"@types/luxon": "^1.25.0",
"@types/mocha": "^8.2.2",
"@types/nanoid": "^3.0.0",
"@types/nodemailer": "^6.4.4",
"@types/oauth": "^0.9.1",
"@types/private-ip": "^1.0.0",
"@types/sanitize-html": "^1.27.1",
"@types/sax": "^1.2.7",
2024-03-12 13:36:50 +00:00
"@types/showdown": "^2.0.6",
"@types/sinon": "^10.0.13",
"@types/sinon-chai": "^3.2.8",
"@types/supertest": "^2.0.11",
"@types/urlsafe-base64": "^1.0.28",
"@types/uuid": "^8.3.0",
"@types/voca": "^1.4.0",
2022-02-11 17:24:33 +00:00
"chai": "^4.3.4",
2022-06-01 02:30:23 +00:00
"chai-as-promised": "^7.1.1",
2022-02-11 17:24:33 +00:00
"chai-string": "^1.5.0",
"circular-dependency-plugin": "^5.2.0",
"mocha": "^9.0.1",
"mocha-unfunk-reporter": "^0.4.0",
2022-02-11 17:24:33 +00:00
"nock": "^13.2.4",
Rebased version of the elastic PR (#225) * Add elastic to our docker compose * add AND/OR/NOT search operations * add elastic and create article in elastic * change error code when elastic throws error * add search pages in elastic * add search by labels * Add elastic to GitHub Action * Update elastic version * Fix port for elastic * add url in search query * Set elastic features when running tests * add debug logs * Use localhost instead of service hostname * refresh elastic after create/update * update search labels query * add typescript support * search pages in elastic * fix search queries * use elastic for saving page * fix test failure * update getArticle api to use elastic * use generic get page function * add elastic migration python script * fix bulk helper param * save elastic page id in article_saving_request instead of postgres article_id * fix page archiving and deleting * add tests for deleteArticle * remove custom date type in elastic mappings which not exist in older version of elastic * fix timestamp format issue * add tests for save reading progress * add tests for save file * optimize search results * add alias to index * update migration script to receive env var as params * Add failing test to validate we don't decrease reading progress This test is failing with Elastic because we aren't fetching the reading progress from elastic here, and are fetching it from postgres. * Rename readingProgress to readingProgressPercent This is the name stored in elastic, so fixes issues pulling the value out. * Linting * Add failing test for creating highlights w/elastic This test fails because the highlight can't be looked up. Is there a different ID we should be passing in to query for highlights, or do we need to update the query to look for elastic_id? * add tests code coverage threshold * update nyc config * include more files in test coverage * change alias name * update updateContent to update pages in elastic * remove debug log * fix createhighlight test * search pages by alias in elastic * update set labels and delete labels in elastic * migration script enumeration * make BULK_SIZE an env var * fix pdf search indexing * debug github action exit issue * call pubsub when create/update/delete page in elastic * fix json parsing bug and reduce reading data from file * replace a depreciated pubsub api call * debug github action exit issue * debug github action exit issue * add handler to upload elastic page data to GCS * fix tests * Use http_auth instead of basic_auth * add index creation and existing postgres tables update in migration script * fix a typo to connect to elastic * rename readingProgress to readingProgressPercent * migrate elastic_page_id in highlights and article_saving_request tables * update migration script to include number of updated rows * update db migration query * read index mappings from file * fix upload pages to gcs * fix tests failure due to pageContext * fix upload file id not exist error * Handle savedAt & isArchived attributes w/out quering elastic * Fix prettier issues * fix content-type mismatching * revert pageId to linkId because frontend was not deployed yet * fix newsletters and attachment not saved in elastic * put linkId in article for setting labels * exclude orginalHtml in the result of searching to improve performace * exclude content in the result of searching to improve performace * remove score sorting * do not refresh immediately to reduce searching and indexing time * do not replace the backup data in gcs * fix no article id defined in articleSavingRequest * add logging of elastic api running time * reduce home feed pagination size to 15 * reduce home feed pagination size to 10 * stop revalidating first page * do not use a separate api to fetch reading progress * Remove unused comment * get reading progress if not exists * replace ngram tokenizer with standard tokenizer * fix tests * remove .env.local * add sort keyword in searching to sort by score Co-authored-by: Hongbo Wu <hongbo@omnivore.app>
2022-03-16 04:08:59 +00:00
"nyc": "^15.1.0",
2022-02-11 17:24:33 +00:00
"postgrator": "^4.2.0",
"sinon": "^14.0.0",
"sinon-chai": "^3.7.0",
2022-02-11 17:24:33 +00:00
"ts-node-dev": "^1.1.8"
},
"engines": {
2023-07-05 17:16:30 +00:00
"node": "18.16.1"
2022-02-11 17:24:33 +00:00
},
"volta": {
"extends": "../../package.json"
2022-02-11 17:24:33 +00:00
}
2024-03-30 02:42:01 +00:00
}