From 795ca47414b18acb8d1f125453fa171ea2c45606 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Tue, 6 Sep 2022 10:19:58 +0800 Subject: [PATCH] Save wordsCount when page is created --- packages/api/package.json | 3 +- packages/api/src/elastic/pages.ts | 2 + packages/api/src/elastic/types.ts | 3 +- .../db/elastic_migrations/index_settings.json | 2 +- yarn.lock | 81 +++++++++++++++++++ 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index 7b275b385..5297027a2 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -86,7 +86,8 @@ "urlsafe-base64": "^1.0.0", "uuid": "^8.3.1", "voca": "^1.4.0", - "winston": "^3.3.3" + "winston": "^3.3.3", + "word-counting": "^1.1.4" }, "devDependencies": { "@babel/register": "^7.14.5", diff --git a/packages/api/src/elastic/pages.ts b/packages/api/src/elastic/pages.ts index 97b00cdf0..a240a42e1 100644 --- a/packages/api/src/elastic/pages.ts +++ b/packages/api/src/elastic/pages.ts @@ -22,6 +22,7 @@ import { import { client, INDEX_ALIAS } from './index' import { EntityType } from '../datalayer/pubsub' import { ResponseError } from '@elastic/elasticsearch/lib/errors' +import wordsCounter from 'word-counting' const appendQuery = (body: SearchBody, query: string): void => { body.query.bool.should.push({ @@ -190,6 +191,7 @@ export const createPage = async ( ...page, updatedAt: new Date(), savedAt: new Date(), + wordsCount: wordsCounter(page.content, { isHtml: true }).wordsCount, }, refresh: ctx.refresh, }) diff --git a/packages/api/src/elastic/types.ts b/packages/api/src/elastic/types.ts index 0f27998c6..33d00b038 100644 --- a/packages/api/src/elastic/types.ts +++ b/packages/api/src/elastic/types.ts @@ -216,7 +216,7 @@ export interface Page { language?: string readAt?: Date listenedAt?: Date - wordCount?: number + wordsCount?: number } export interface SearchItem { @@ -246,6 +246,7 @@ export interface SearchItem { updatedAt?: Date labels?: Label[] highlights?: Highlight[] + wordsCount?: number } const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const diff --git a/packages/db/elastic_migrations/index_settings.json b/packages/db/elastic_migrations/index_settings.json index 3c30c357d..61ec732d4 100644 --- a/packages/db/elastic_migrations/index_settings.json +++ b/packages/db/elastic_migrations/index_settings.json @@ -152,7 +152,7 @@ "listenedAt": { "type": "date" }, - "wordCount": { + "wordsCount": { "type": "integer" } } diff --git a/yarn.lock b/yarn.lock index 09a7d55a3..c7a25aa2b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5097,6 +5097,14 @@ dset "^3.1.1" tiny-hashes "^1.0.1" +"@selderee/plugin-htmlparser2@^0.6.0": + version "0.6.0" + resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.6.0.tgz#27e994afd1c2cb647ceb5406a185a5574188069d" + integrity sha512-J3jpy002TyBjd4N/p6s+s90eX42H2eRhK3SbsZuvTDv977/E8p2U3zikdiehyJja66do7FlxLomZLPlvl2/xaA== + dependencies: + domhandler "^4.2.0" + selderee "^0.6.0" + "@sendgrid/client@^7.6.0", "@sendgrid/client@^7.7.0": version "7.7.0" resolved "https://registry.yarnpkg.com/@sendgrid/client/-/client-7.7.0.tgz#f8f67abd604205a0d0b1af091b61517ef465fdbf" @@ -12274,6 +12282,11 @@ dir-glob@^3.0.1: dependencies: path-type "^4.0.0" +discontinuous-range@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/discontinuous-range/-/discontinuous-range-1.0.0.tgz#e38331f0844bba49b9a9cb71c771585aab1bc65a" + integrity sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ== + dlv@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79" @@ -15211,6 +15224,18 @@ html-tags@^3.1.0: resolved "https://registry.yarnpkg.com/html-tags/-/html-tags-3.2.0.tgz#dbb3518d20b726524e4dd43de397eb0a95726961" integrity sha512-vy7ClnArOZwCnqZgvv+ddgHgJiAFXe3Ge9ML5/mBctVJoUoYPCdxVucOywjDARn6CVoh3dRSFdPHy2sX80L0Wg== +html-to-text@^8.1.0: + version "8.2.1" + resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-8.2.1.tgz#4a75b8a1b646149bd71c50527adb568990bf459b" + integrity sha512-aN/3JvAk8qFsWVeE9InWAWueLXrbkoVZy0TkzaGhoRBC2gCFEeRLDDJN3/ijIGHohy6H+SZzUQWN/hcYtaPK8w== + dependencies: + "@selderee/plugin-htmlparser2" "^0.6.0" + deepmerge "^4.2.2" + he "^1.2.0" + htmlparser2 "^6.1.0" + minimist "^1.2.6" + selderee "^0.6.0" + html-void-elements@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-1.0.5.tgz#ce9159494e86d95e45795b166c2021c2cfca4483" @@ -18771,6 +18796,11 @@ module-details-from-path@^1.0.3: resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.3.tgz#114c949673e2a8a35e9d35788527aa37b679da2b" integrity sha1-EUyUlnPiqKNenTV4hSeqN7Z52is= +moo@^0.5.0, moo@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.1.tgz#7aae7f384b9b09f620b6abf6f74ebbcd1b65dbc4" + integrity sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w== + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" @@ -18888,6 +18918,16 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= +nearley@^2.20.1: + version "2.20.1" + resolved "https://registry.yarnpkg.com/nearley/-/nearley-2.20.1.tgz#246cd33eff0d012faf197ff6774d7ac78acdd474" + integrity sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ== + dependencies: + commander "^2.19.0" + moo "^0.5.0" + railroad-diagrams "^1.0.0" + randexp "0.4.6" + negotiator@0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" @@ -20139,6 +20179,14 @@ parse5@^5.1.1: resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178" integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug== +parseley@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.7.0.tgz#9949e3a0ed05c5072adb04f013c2810cf49171a8" + integrity sha512-xyOytsdDu077M3/46Am+2cGXEKM9U9QclBDv7fimY7e+BBlxh2JcBp2mgNsmkyA9uvgyTjVzDi7cP1v4hcFxbw== + dependencies: + moo "^0.5.1" + nearley "^2.20.1" + parseurl@^1.3.3, parseurl@~1.3.2, parseurl@~1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" @@ -21152,11 +21200,24 @@ quick-lru@^4.0.1: resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== +railroad-diagrams@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz#eb7e6267548ddedfb899c1b90e57374559cddb7e" + integrity sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A== + ramda@^0.21.0: version "0.21.0" resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.21.0.tgz#a001abedb3ff61077d4ff1d577d44de77e8d0a35" integrity sha1-oAGr7bP/YQd9T/HVd9RN536NCjU= +randexp@0.4.6: + version "0.4.6" + resolved "https://registry.yarnpkg.com/randexp/-/randexp-0.4.6.tgz#e986ad5e5e31dae13ddd6f7b3019aa7c87f60ca3" + integrity sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ== + dependencies: + discontinuous-range "1.0.0" + ret "~0.1.10" + randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -22329,6 +22390,13 @@ secure-json-parse@^2.3.1: resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85" integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg== +selderee@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7" + integrity sha512-ibqWGV5aChDvfVdqNYuaJP/HnVBhlRGSRrlbttmlMpHcLuTqqbMH36QkSs9GEgj5M88JDYLI8eyP94JaQ8xRlg== + dependencies: + parseley "^0.7.0" + select-hose@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" @@ -25360,6 +25428,19 @@ winston@^3.3.3: triple-beam "^1.3.0" winston-transport "^4.5.0" +word-counting@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/word-counting/-/word-counting-1.1.4.tgz#4d772df20bd86e2e8b00596c8e1ab2f355578ddd" + integrity sha512-SsAKEoa6FzQTV7fR27vDOHO9m2f7cnGhppP+e0c6JCn+pDg88kCMVfrt0Qr8XcbbW2o6HIum4yFX8WEnxZ5xLA== + dependencies: + html-to-text "^8.1.0" + word-regex "^0.1.2" + +word-regex@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/word-regex/-/word-regex-0.1.2.tgz#a3bc7f2d222ce4a93c246c3ef69458f61f511639" + integrity sha512-4jK/OibPeindR9o/sryObhVWNgD2LJCMJFWEME69p48sEYpE9axfyjHK+RqYcOeoEoqcqJEPE9iMdiiFpXHo0Q== + word-wrap@^1.2.3, word-wrap@~1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"