prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Save wordsCount when page is created

Browse source
This commit is contained in:
Hongbo Wu 2022-09-06 10:19:58 +08:00
parent d00391b593
commit 795ca47414
5 changed files with 88 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/api/package.json
View file

@ -86,7 +86,8 @@
"urlsafe-base64": "^1.0.0",
"uuid": "^8.3.1",
"voca": "^1.4.0",
"winston": "^3.3.3"
"winston": "^3.3.3",
"word-counting": "^1.1.4"
},
"devDependencies": {
"@babel/register": "^7.14.5",

2
packages/api/src/elastic/pages.ts
View file

@ -22,6 +22,7 @@ import {
import { client, INDEX_ALIAS } from './index'
import { EntityType } from '../datalayer/pubsub'
import { ResponseError } from '@elastic/elasticsearch/lib/errors'
import wordsCounter from 'word-counting'
const appendQuery = (body: SearchBody, query: string): void => {
body.query.bool.should.push({
@ -190,6 +191,7 @@ export const createPage = async (
...page,
updatedAt: new Date(),
savedAt: new Date(),
wordsCount: wordsCounter(page.content, { isHtml: true }).wordsCount,
},
refresh: ctx.refresh,
})

3
packages/api/src/elastic/types.ts
View file

@ -216,7 +216,7 @@ export interface Page {
language?: string
readAt?: Date
listenedAt?: Date
wordCount?: number
wordsCount?: number
}
export interface SearchItem {
@ -246,6 +246,7 @@ export interface SearchItem {
updatedAt?: Date
labels?: Label[]
highlights?: Highlight[]
wordsCount?: number
}
const keys = ['_id', 'url', 'slug', 'userId', 'uploadFileId', 'state'] as const

2
packages/db/elastic_migrations/index_settings.json
View file

@ -152,7 +152,7 @@
"listenedAt": {
"type": "date"
},
"wordCount": {
"wordsCount": {
"type": "integer"
}
}

81
yarn.lock
View file

@ -5097,6 +5097,14 @@
dset "^3.1.1"
tiny-hashes "^1.0.1"
"@selderee/plugin-htmlparser2@^0.6.0":
version "0.6.0"
resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.6.0.tgz#27e994afd1c2cb647ceb5406a185a5574188069d"
integrity sha512-J3jpy002TyBjd4N/p6s+s90eX42H2eRhK3SbsZuvTDv977/E8p2U3zikdiehyJja66do7FlxLomZLPlvl2/xaA==
dependencies:
domhandler "^4.2.0"
selderee "^0.6.0"
"@sendgrid/client@^7.6.0", "@sendgrid/client@^7.7.0":
version "7.7.0"
resolved "https://registry.yarnpkg.com/@sendgrid/client/-/client-7.7.0.tgz#f8f67abd604205a0d0b1af091b61517ef465fdbf"
@ -12274,6 +12282,11 @@ dir-glob@^3.0.1:
dependencies:
path-type "^4.0.0"
discontinuous-range@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/discontinuous-range/-/discontinuous-range-1.0.0.tgz#e38331f0844bba49b9a9cb71c771585aab1bc65a"
integrity sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ==
dlv@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79"
@ -15211,6 +15224,18 @@ html-tags@^3.1.0:
resolved "https://registry.yarnpkg.com/html-tags/-/html-tags-3.2.0.tgz#dbb3518d20b726524e4dd43de397eb0a95726961"
integrity sha512-vy7ClnArOZwCnqZgvv+ddgHgJiAFXe3Ge9ML5/mBctVJoUoYPCdxVucOywjDARn6CVoh3dRSFdPHy2sX80L0Wg==
html-to-text@^8.1.0:
version "8.2.1"
resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-8.2.1.tgz#4a75b8a1b646149bd71c50527adb568990bf459b"
integrity sha512-aN/3JvAk8qFsWVeE9InWAWueLXrbkoVZy0TkzaGhoRBC2gCFEeRLDDJN3/ijIGHohy6H+SZzUQWN/hcYtaPK8w==
dependencies:
"@selderee/plugin-htmlparser2" "^0.6.0"
deepmerge "^4.2.2"
he "^1.2.0"
htmlparser2 "^6.1.0"
minimist "^1.2.6"
selderee "^0.6.0"
html-void-elements@^1.0.0:
version "1.0.5"
resolved "https://registry.yarnpkg.com/html-void-elements/-/html-void-elements-1.0.5.tgz#ce9159494e86d95e45795b166c2021c2cfca4483"
@ -18771,6 +18796,11 @@ module-details-from-path@^1.0.3:
resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.3.tgz#114c949673e2a8a35e9d35788527aa37b679da2b"
integrity sha1-EUyUlnPiqKNenTV4hSeqN7Z52is=
moo@^0.5.0, moo@^0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.1.tgz#7aae7f384b9b09f620b6abf6f74ebbcd1b65dbc4"
integrity sha512-I1mnb5xn4fO80BH9BLcF0yLypy2UKl+Cb01Fu0hJRkJjlCRtxZMWkTdAtDd5ZqCOxtCkhmRwyI57vWT+1iZ67w==
move-concurrently@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92"
@ -18888,6 +18918,16 @@ natural-compare@^1.4.0:
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=
nearley@^2.20.1:
version "2.20.1"
resolved "https://registry.yarnpkg.com/nearley/-/nearley-2.20.1.tgz#246cd33eff0d012faf197ff6774d7ac78acdd474"
integrity sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==
dependencies:
commander "^2.19.0"
moo "^0.5.0"
railroad-diagrams "^1.0.0"
randexp "0.4.6"
negotiator@0.6.3:
version "0.6.3"
resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd"
@ -20139,6 +20179,14 @@ parse5@^5.1.1:
resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178"
integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==
parseley@^0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.7.0.tgz#9949e3a0ed05c5072adb04f013c2810cf49171a8"
integrity sha512-xyOytsdDu077M3/46Am+2cGXEKM9U9QclBDv7fimY7e+BBlxh2JcBp2mgNsmkyA9uvgyTjVzDi7cP1v4hcFxbw==
dependencies:
moo "^0.5.1"
nearley "^2.20.1"
parseurl@^1.3.3, parseurl@~1.3.2, parseurl@~1.3.3:
version "1.3.3"
resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4"
@ -21152,11 +21200,24 @@ quick-lru@^4.0.1:
resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f"
integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==
railroad-diagrams@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz#eb7e6267548ddedfb899c1b90e57374559cddb7e"
integrity sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==
ramda@^0.21.0:
version "0.21.0"
resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.21.0.tgz#a001abedb3ff61077d4ff1d577d44de77e8d0a35"
integrity sha1-oAGr7bP/YQd9T/HVd9RN536NCjU=
randexp@0.4.6:
version "0.4.6"
resolved "https://registry.yarnpkg.com/randexp/-/randexp-0.4.6.tgz#e986ad5e5e31dae13ddd6f7b3019aa7c87f60ca3"
integrity sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==
dependencies:
discontinuous-range "1.0.0"
ret "~0.1.10"
randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a"
@ -22329,6 +22390,13 @@ secure-json-parse@^2.3.1:
resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.4.0.tgz#5aaeaaef85c7a417f76271a4f5b0cc3315ddca85"
integrity sha512-Q5Z/97nbON5t/L/sH6mY2EacfjVGwrCcSi5D3btRO2GZ8pf1K1UN7Z9H5J57hjVU2Qzxr1xO+FmBhOvEkzCMmg==
selderee@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.6.0.tgz#f3bee66cfebcb6f33df98e4a1df77388b42a96f7"
integrity sha512-ibqWGV5aChDvfVdqNYuaJP/HnVBhlRGSRrlbttmlMpHcLuTqqbMH36QkSs9GEgj5M88JDYLI8eyP94JaQ8xRlg==
dependencies:
parseley "^0.7.0"
select-hose@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca"
@ -25360,6 +25428,19 @@ winston@^3.3.3:
triple-beam "^1.3.0"
winston-transport "^4.5.0"
word-counting@^1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/word-counting/-/word-counting-1.1.4.tgz#4d772df20bd86e2e8b00596c8e1ab2f355578ddd"
integrity sha512-SsAKEoa6FzQTV7fR27vDOHO9m2f7cnGhppP+e0c6JCn+pDg88kCMVfrt0Qr8XcbbW2o6HIum4yFX8WEnxZ5xLA==
dependencies:
html-to-text "^8.1.0"
word-regex "^0.1.2"
word-regex@^0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/word-regex/-/word-regex-0.1.2.tgz#a3bc7f2d222ce4a93c246c3ef69458f61f511639"
integrity sha512-4jK/OibPeindR9o/sryObhVWNgD2LJCMJFWEME69p48sEYpE9axfyjHK+RqYcOeoEoqcqJEPE9iMdiiFpXHo0Q==
word-wrap@^1.2.3, word-wrap@~1.2.3:
version "1.2.3"
resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c"