mirror of
https://github.com/omnivore-app/omnivore.git
synced 2026-03-11 08:54:26 +00:00
fix typo
This commit is contained in:
Hongbo Wu
parent
3f572ea89a
commit
4a55e801b4
13 changed files with 185 additions and 1289 deletions
|
|
@ -12,7 +12,6 @@
|
|||
"copy-files": "copyfiles -u 1 src/**/*.html dist/"
|
||||
},
|
||||
"dependencies": {
|
||||
"@elastic/elasticsearch": "~7.12.0",
|
||||
"@google-cloud/logging-winston": "^6.0.0",
|
||||
"@google-cloud/monitoring": "^4.0.0",
|
||||
"@google-cloud/opentelemetry-cloud-trace-exporter": "^2.0.0",
|
||||
|
|
@ -55,7 +54,6 @@
|
|||
"dompurify": "^2.0.17",
|
||||
"dot-case": "^3.0.4",
|
||||
"dotenv": "^8.2.0",
|
||||
"elastic-ts": "^0.9.0",
|
||||
"express": "^4.17.1",
|
||||
"express-http-context2": "^1.0.0",
|
||||
"express-rate-limit": "^6.3.0",
|
||||
|
|
|
|||
|
|
@ -1,974 +0,0 @@
|
|||
import { errors } from '@elastic/elasticsearch'
|
||||
import { BuiltQuery, ESBuilder, esBuilder } from 'elastic-ts'
|
||||
import { BulkActionType } from '../generated/graphql'
|
||||
import { EntityType } from '../pubsub'
|
||||
import { wordsCount } from '../utils/helpers'
|
||||
import {
|
||||
DateFilter,
|
||||
FieldFilter,
|
||||
HasFilter,
|
||||
InFilter,
|
||||
LabelFilter,
|
||||
LabelFilterType,
|
||||
NoFilter,
|
||||
ReadFilter,
|
||||
SortBy,
|
||||
SortOrder,
|
||||
} from '../utils/search'
|
||||
import { client, INDEX_ALIAS, logger } from './index'
|
||||
import {
|
||||
ArticleSavingRequestStatus,
|
||||
Label,
|
||||
Page,
|
||||
PageContext,
|
||||
PageSearchArgs,
|
||||
PageType,
|
||||
ParamSet,
|
||||
SearchResponse,
|
||||
} from './types'
|
||||
|
||||
const MAX_CONTENT_LENGTH = 5 * 1024 * 1024 // 5MB and 10MB for both content and originalHtml
|
||||
const CONTENT_LENGTH_ERROR = 'Your page content is too large to be saved.'
|
||||
|
||||
const appendQuery = (builder: ESBuilder, query: string): ESBuilder => {
|
||||
interface Field {
|
||||
field: string
|
||||
boost: number
|
||||
}
|
||||
|
||||
const wildcardQuery = (field: Field) => {
|
||||
return {
|
||||
[field.field]: {
|
||||
value: query,
|
||||
case_insensitive: true,
|
||||
boost: field.boost,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// add boost to the field name like title^3
|
||||
const fieldWithBoost = (field: Field) =>
|
||||
`${field.field}${field.boost > 1 ? `^${field.boost}` : ''}`
|
||||
|
||||
// get the parent field name like highlights from highlights.annotation
|
||||
const getParentField = (nestedField: string) => nestedField.split('.')[0]
|
||||
|
||||
const nonNestedFields: Field[] = [
|
||||
{ field: 'title', boost: 3 },
|
||||
{ field: 'content', boost: 1 },
|
||||
{ field: 'author', boost: 1 },
|
||||
{ field: 'description', boost: 1 },
|
||||
{ field: 'siteName', boost: 2 },
|
||||
]
|
||||
const nestedFields: Field[] = [{ field: 'highlights.annotation', boost: 2 }]
|
||||
|
||||
// minimum_should_match: 1 means that at least one of the queries must match
|
||||
builder = builder.queryMinimumShouldMatch(1)
|
||||
|
||||
// wildcard query
|
||||
if (query.includes('*')) {
|
||||
nonNestedFields.forEach((field) => {
|
||||
builder = builder.orQuery('wildcard', wildcardQuery(field))
|
||||
})
|
||||
|
||||
nestedFields.forEach((nestedField) => {
|
||||
builder = builder.orQuery('nested', {
|
||||
path: getParentField(nestedField.field),
|
||||
query: {
|
||||
wildcard: wildcardQuery(nestedField),
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
return builder
|
||||
}
|
||||
|
||||
// match query
|
||||
builder = builder.orQuery('multi_match', {
|
||||
query,
|
||||
fields: nonNestedFields.map((field) => fieldWithBoost(field)),
|
||||
type: 'best_fields',
|
||||
tie_breaker: 0.3,
|
||||
operator: 'and',
|
||||
})
|
||||
|
||||
nestedFields.forEach((nestedField) => {
|
||||
builder = builder.orQuery('nested', {
|
||||
path: getParentField(nestedField.field),
|
||||
query: {
|
||||
match: {
|
||||
[nestedField.field]: {
|
||||
query,
|
||||
boost: nestedField.boost,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendTypeFilter = (builder: ESBuilder, filter: PageType): ESBuilder => {
|
||||
return builder.query('term', { pageType: filter })
|
||||
}
|
||||
|
||||
const appendReadFilter = (
|
||||
builder: ESBuilder,
|
||||
filter: ReadFilter
|
||||
): ESBuilder => {
|
||||
switch (filter) {
|
||||
case ReadFilter.UNREAD:
|
||||
return builder.query('range', {
|
||||
readingProgressPercent: {
|
||||
lt: 98,
|
||||
},
|
||||
})
|
||||
case ReadFilter.READ:
|
||||
return builder.query('range', {
|
||||
readingProgressPercent: {
|
||||
gte: 98,
|
||||
},
|
||||
})
|
||||
}
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendInFilter = (builder: ESBuilder, filter: InFilter): ESBuilder => {
|
||||
switch (filter) {
|
||||
case InFilter.ARCHIVE:
|
||||
return builder.query('exists', { field: 'archivedAt' })
|
||||
case InFilter.INBOX:
|
||||
return builder.notQuery('exists', { field: 'archivedAt' })
|
||||
case InFilter.TRASH:
|
||||
// return only deleted pages within 14 days
|
||||
return builder
|
||||
.query('term', {
|
||||
state: ArticleSavingRequestStatus.Deleted,
|
||||
})
|
||||
.andQuery('range', {
|
||||
updatedAt: {
|
||||
gte: 'now-14d',
|
||||
},
|
||||
})
|
||||
case InFilter.LIBRARY:
|
||||
return builder
|
||||
.query('bool', {
|
||||
should: [
|
||||
{
|
||||
nested: {
|
||||
path: 'labels',
|
||||
query: {
|
||||
term: {
|
||||
'labels.name': 'library',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
bool: {
|
||||
must_not: [
|
||||
{
|
||||
nested: {
|
||||
path: 'labels',
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': ['newsletter', 'rss'],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
should: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
})
|
||||
.notQuery('exists', { field: 'archivedAt' })
|
||||
case InFilter.SUBSCRIPTION:
|
||||
return builder
|
||||
.andQuery('nested', {
|
||||
path: 'labels',
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': ['newsletter', 'rss'],
|
||||
},
|
||||
},
|
||||
})
|
||||
.notQuery('nested', {
|
||||
path: 'labels',
|
||||
query: {
|
||||
term: {
|
||||
'labels.name': 'library',
|
||||
},
|
||||
},
|
||||
})
|
||||
.notQuery('exists', { field: 'archivedAt' })
|
||||
default:
|
||||
return builder
|
||||
}
|
||||
}
|
||||
|
||||
const appendHasFilters = (
|
||||
builder: ESBuilder,
|
||||
filters: HasFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
switch (filter) {
|
||||
case HasFilter.HIGHLIGHTS:
|
||||
builder = builder.query('nested', {
|
||||
path: 'highlights',
|
||||
query: {
|
||||
exists: {
|
||||
field: 'highlights',
|
||||
},
|
||||
},
|
||||
})
|
||||
break
|
||||
case HasFilter.LABELS:
|
||||
builder = builder.query('nested', {
|
||||
path: 'labels',
|
||||
query: {
|
||||
exists: {
|
||||
field: 'labels',
|
||||
},
|
||||
},
|
||||
})
|
||||
break
|
||||
}
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendExcludeLabelFilter = (
|
||||
builder: ESBuilder,
|
||||
filters: LabelFilter[]
|
||||
): ESBuilder => {
|
||||
const labels = filters.map((filter) => filter.labels).flat()
|
||||
return builder.notQuery('nested', {
|
||||
path: 'labels',
|
||||
query: {
|
||||
terms: {
|
||||
'labels.name': labels,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const appendIncludeLabelFilter = (
|
||||
builder: ESBuilder,
|
||||
filters: LabelFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
builder = builder.query('nested', {
|
||||
path: 'labels',
|
||||
query: {
|
||||
bool: {
|
||||
should: filter.labels.map((label) => {
|
||||
if (label.includes('*')) {
|
||||
// Wildcard query
|
||||
return {
|
||||
wildcard: {
|
||||
'labels.name': {
|
||||
value: label,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
return {
|
||||
term: {
|
||||
'labels.name': label,
|
||||
},
|
||||
}
|
||||
}),
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendDateFilters = (
|
||||
builder: ESBuilder,
|
||||
filters: DateFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
builder = builder.query('range', {
|
||||
[filter.field]: {
|
||||
gt: filter.startDate?.getTime(),
|
||||
lt: filter.endDate?.getTime(),
|
||||
},
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendTermFilters = (
|
||||
builder: ESBuilder,
|
||||
filters: FieldFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
builder = builder.query('term', {
|
||||
[filter.field]: filter.value,
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendMatchFilters = (
|
||||
builder: ESBuilder,
|
||||
filters: FieldFilter[]
|
||||
): ESBuilder => {
|
||||
filters.forEach((filter) => {
|
||||
if (filter.nested) {
|
||||
// nested query
|
||||
builder = builder.query('nested', {
|
||||
path: filter.field.split('.')[0], // get the nested field name
|
||||
query: {
|
||||
match: {
|
||||
[filter.field]: filter.value,
|
||||
},
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
builder = builder.query('match', {
|
||||
[filter.field]: filter.value,
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendIdsFilter = (builder: ESBuilder, ids: string[]): ESBuilder => {
|
||||
return builder.query('terms', {
|
||||
_id: ids,
|
||||
})
|
||||
}
|
||||
|
||||
const appendRecommendedBy = (
|
||||
builder: ESBuilder,
|
||||
recommendedBy: string
|
||||
): ESBuilder => {
|
||||
const query =
|
||||
recommendedBy === '*'
|
||||
? {
|
||||
exists: {
|
||||
field: 'recommendations',
|
||||
},
|
||||
}
|
||||
: {
|
||||
term: {
|
||||
'recommendations.name': recommendedBy,
|
||||
},
|
||||
}
|
||||
return builder.query('nested', {
|
||||
path: 'recommendations',
|
||||
query,
|
||||
})
|
||||
}
|
||||
|
||||
const appendNoFilters = (
|
||||
builder: ESBuilder,
|
||||
noFilters: NoFilter[]
|
||||
): ESBuilder => {
|
||||
noFilters.forEach((filter) => {
|
||||
builder = builder.notQuery('nested', {
|
||||
path: filter.field,
|
||||
query: {
|
||||
exists: {
|
||||
field: filter.field,
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
return builder
|
||||
}
|
||||
|
||||
const appendSiteNameFilter = (
|
||||
builder: ESBuilder,
|
||||
siteName: string
|
||||
): ESBuilder => {
|
||||
return builder.query('bool', {
|
||||
should: [
|
||||
{
|
||||
match: {
|
||||
siteName,
|
||||
},
|
||||
},
|
||||
{
|
||||
wildcard: {
|
||||
// siteName is a domain name, so we need to wildcard the end
|
||||
url: `*${siteName}*`,
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
})
|
||||
}
|
||||
|
||||
export const createPage = async (
|
||||
page: Page,
|
||||
ctx: PageContext
|
||||
): Promise<string | undefined> => {
|
||||
try {
|
||||
if (page.content.length > MAX_CONTENT_LENGTH) {
|
||||
logger.info('page content is too large', {
|
||||
pageId: page.id,
|
||||
contentLength: page.content.length,
|
||||
})
|
||||
|
||||
page.content = CONTENT_LENGTH_ERROR
|
||||
}
|
||||
|
||||
const { body } = await client.index({
|
||||
id: page.id || undefined,
|
||||
index: INDEX_ALIAS,
|
||||
body: {
|
||||
...page,
|
||||
updatedAt: new Date(),
|
||||
savedAt: page.savedAt || new Date(),
|
||||
wordsCount: page.wordsCount ?? wordsCount(page.content),
|
||||
},
|
||||
refresh: 'wait_for', // wait for the index to be refreshed before returning
|
||||
})
|
||||
|
||||
page.id = body._id as string
|
||||
|
||||
const shouldPublish = ctx.shouldPublish ?? true
|
||||
// only publish a pubsub event if we should
|
||||
if (shouldPublish) {
|
||||
await ctx.pubsub?.entityCreated<Page>(EntityType.PAGE, page, ctx.uid)
|
||||
}
|
||||
|
||||
return page.id
|
||||
} catch (e) {
|
||||
logger.error('failed to create a page in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export const updatePage = async (
|
||||
id: string,
|
||||
page: Partial<Page>,
|
||||
ctx: PageContext
|
||||
): Promise<boolean> => {
|
||||
try {
|
||||
if (page.content && page.content.length > MAX_CONTENT_LENGTH) {
|
||||
logger.info('page content is too large', {
|
||||
pageId: page.id,
|
||||
contentLength: page.content.length,
|
||||
})
|
||||
|
||||
page.content = CONTENT_LENGTH_ERROR
|
||||
}
|
||||
|
||||
await client.update({
|
||||
index: INDEX_ALIAS,
|
||||
id,
|
||||
body: {
|
||||
doc: {
|
||||
...page,
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
},
|
||||
refresh: ctx.refresh,
|
||||
retry_on_conflict: 3,
|
||||
})
|
||||
|
||||
if (page.state === ArticleSavingRequestStatus.Deleted) {
|
||||
await ctx.pubsub.entityDeleted(EntityType.PAGE, id, ctx.uid)
|
||||
return true
|
||||
}
|
||||
|
||||
await ctx.pubsub.entityUpdated<Partial<Page>>(
|
||||
EntityType.PAGE,
|
||||
{ ...page, id },
|
||||
ctx.uid
|
||||
)
|
||||
return true
|
||||
} catch (e) {
|
||||
if (
|
||||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
logger.info('page has been deleted', id)
|
||||
return false
|
||||
}
|
||||
logger.error('failed to update a page in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export const deletePage = async (
|
||||
id: string,
|
||||
ctx: PageContext
|
||||
): Promise<boolean> => {
|
||||
try {
|
||||
const { body } = await client.delete({
|
||||
index: INDEX_ALIAS,
|
||||
id,
|
||||
refresh: ctx.refresh,
|
||||
})
|
||||
|
||||
return body.deleted !== 0
|
||||
} catch (e) {
|
||||
if (
|
||||
e instanceof errors.ResponseError &&
|
||||
e.message === 'document_missing_exception'
|
||||
) {
|
||||
logger.info('page has been deleted', id)
|
||||
return false
|
||||
}
|
||||
logger.error('failed to delete a page in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export const getPageByParam = async <K extends keyof ParamSet>(
|
||||
params: Record<K, ParamSet[K] | ParamSet[K][]>,
|
||||
includeOriginalHtml = false
|
||||
): Promise<Page | undefined> => {
|
||||
try {
|
||||
let builder = esBuilder()
|
||||
.size(1)
|
||||
.rawOption('_source', {
|
||||
excludes: includeOriginalHtml ? [] : ['originalHtml'],
|
||||
})
|
||||
// filter out undefined and null values and empty arrays
|
||||
// and build the query
|
||||
Object.entries<ParamSet[K] | ParamSet[K][]>(params)
|
||||
.filter(
|
||||
([, value]) =>
|
||||
value != null && !(Array.isArray(value) && value.length === 0)
|
||||
)
|
||||
.forEach(([key, value]) => {
|
||||
Array.isArray(value)
|
||||
? (builder = builder.query('terms', key, value))
|
||||
: (builder = builder.query('term', key, value))
|
||||
})
|
||||
const { body } = await client.search<SearchResponse<Page>>({
|
||||
index: INDEX_ALIAS,
|
||||
body: builder.build(),
|
||||
track_total_hits: true,
|
||||
})
|
||||
|
||||
if (body.hits.total.value === 0) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
return {
|
||||
...body.hits.hits[0]._source,
|
||||
id: body.hits.hits[0]._id,
|
||||
} as Page
|
||||
} catch (e) {
|
||||
logger.error('failed to get page by param in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export const getPageById = async (id: string): Promise<Page | undefined> => {
|
||||
try {
|
||||
if (!id) return undefined
|
||||
|
||||
const { body } = await client.get({
|
||||
index: INDEX_ALIAS,
|
||||
id,
|
||||
})
|
||||
|
||||
return {
|
||||
...body._source,
|
||||
id: body._id as string,
|
||||
} as Page
|
||||
} catch (e) {
|
||||
if (e instanceof errors.ResponseError && e.statusCode === 404) {
|
||||
logger.info('page has been deleted', id)
|
||||
return undefined
|
||||
}
|
||||
logger.error('failed to get page by id in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
const buildSearchBody = (userId: string, args: PageSearchArgs) => {
|
||||
const {
|
||||
query,
|
||||
readFilter = ReadFilter.ALL,
|
||||
typeFilter,
|
||||
labelFilters,
|
||||
inFilter = InFilter.ALL,
|
||||
hasFilters,
|
||||
dateFilters,
|
||||
termFilters,
|
||||
matchFilters,
|
||||
ids,
|
||||
noFilters,
|
||||
siteName,
|
||||
} = args
|
||||
|
||||
const includeLabels = labelFilters?.filter(
|
||||
(filter) => filter.type === LabelFilterType.INCLUDE
|
||||
)
|
||||
const excludeLabels = labelFilters?.filter(
|
||||
(filter) => filter.type === LabelFilterType.EXCLUDE
|
||||
)
|
||||
|
||||
// start building the query
|
||||
let builder = esBuilder().query('term', { userId })
|
||||
|
||||
// append filters
|
||||
if (query) {
|
||||
builder = appendQuery(builder, query)
|
||||
}
|
||||
if (typeFilter) {
|
||||
builder = appendTypeFilter(builder, typeFilter)
|
||||
}
|
||||
if (inFilter !== InFilter.ALL) {
|
||||
builder = appendInFilter(builder, inFilter)
|
||||
}
|
||||
if (readFilter !== ReadFilter.ALL) {
|
||||
builder = appendReadFilter(builder, readFilter)
|
||||
}
|
||||
if (hasFilters && hasFilters.length > 0) {
|
||||
builder = appendHasFilters(builder, hasFilters)
|
||||
}
|
||||
if (includeLabels && includeLabels.length > 0) {
|
||||
builder = appendIncludeLabelFilter(builder, includeLabels)
|
||||
}
|
||||
if (excludeLabels && excludeLabels.length > 0) {
|
||||
builder = appendExcludeLabelFilter(builder, excludeLabels)
|
||||
}
|
||||
if (dateFilters && dateFilters.length > 0) {
|
||||
builder = appendDateFilters(builder, dateFilters)
|
||||
}
|
||||
if (termFilters) {
|
||||
builder = appendTermFilters(builder, termFilters)
|
||||
}
|
||||
if (matchFilters) {
|
||||
builder = appendMatchFilters(builder, matchFilters)
|
||||
}
|
||||
if (ids && ids.length > 0) {
|
||||
builder = appendIdsFilter(builder, ids)
|
||||
}
|
||||
if (args.recommendedBy) {
|
||||
builder = appendRecommendedBy(builder, args.recommendedBy)
|
||||
}
|
||||
if (!args.includePending) {
|
||||
builder = builder.notQuery('term', {
|
||||
state: ArticleSavingRequestStatus.Processing,
|
||||
})
|
||||
}
|
||||
if (!args.includeDeleted && inFilter !== InFilter.TRASH) {
|
||||
builder = builder.notQuery('term', {
|
||||
state: ArticleSavingRequestStatus.Deleted,
|
||||
})
|
||||
}
|
||||
if (noFilters) {
|
||||
builder = appendNoFilters(builder, noFilters)
|
||||
}
|
||||
if (siteName) {
|
||||
builder = appendSiteNameFilter(builder, siteName)
|
||||
}
|
||||
|
||||
return builder
|
||||
}
|
||||
|
||||
export const searchPages = async (
|
||||
args: PageSearchArgs,
|
||||
userId: string
|
||||
): Promise<[Page[], number] | undefined> => {
|
||||
try {
|
||||
const { from = 0, size = 10, sort, includeContent } = args
|
||||
|
||||
// default order is descending
|
||||
const sortOrder = sort?.order || SortOrder.DESCENDING
|
||||
// default sort by saved_at
|
||||
const sortField = sort?.by || SortBy.SAVED
|
||||
|
||||
// build the query
|
||||
const builder = buildSearchBody(userId, args)
|
||||
const body = builder
|
||||
.sort('_score', 'desc') // sort by score first
|
||||
.sort(sortField, sortOrder)
|
||||
.from(from)
|
||||
.size(size)
|
||||
.rawOption('_source', {
|
||||
excludes: includeContent ? [] : ['originalHtml', 'content'],
|
||||
})
|
||||
.build()
|
||||
|
||||
logger.info('searching pages in elastic', body)
|
||||
const response = await client.search<SearchResponse<Page>, BuiltQuery>({
|
||||
index: INDEX_ALIAS,
|
||||
body,
|
||||
})
|
||||
if (response.body.hits.total.value === 0) {
|
||||
return [[], 0]
|
||||
}
|
||||
|
||||
return [
|
||||
response.body.hits.hits.map((hit: { _source: Page; _id: string }) => ({
|
||||
...hit._source,
|
||||
content: includeContent ? hit._source.content : '',
|
||||
id: hit._id,
|
||||
})),
|
||||
response.body.hits.total.value,
|
||||
]
|
||||
} catch (e) {
|
||||
if (e instanceof errors.ResponseError) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
logger.error('failed to search pages in elastic', e.meta.body.error)
|
||||
return undefined
|
||||
}
|
||||
logger.error('failed to search pages in elastic', e)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
export const countByCreatedAt = async (
|
||||
userId: string,
|
||||
from?: number,
|
||||
to?: number
|
||||
): Promise<number> => {
|
||||
try {
|
||||
const { body } = await client.count({
|
||||
index: INDEX_ALIAS,
|
||||
body: {
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
term: {
|
||||
userId,
|
||||
},
|
||||
},
|
||||
{
|
||||
range: {
|
||||
createdAt: {
|
||||
gte: from,
|
||||
lte: to,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
return body.count as number
|
||||
} catch (e) {
|
||||
logger.error('failed to count pages in elastic', e)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
export const deletePagesByParam = async <K extends keyof ParamSet>(
|
||||
param: Record<K, ParamSet[K]>,
|
||||
ctx: PageContext
|
||||
): Promise<boolean> => {
|
||||
try {
|
||||
const params = {
|
||||
query: {
|
||||
bool: {
|
||||
filter: Object.keys(param).map((key) => {
|
||||
return {
|
||||
term: {
|
||||
[key]: param[key as K],
|
||||
},
|
||||
}
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const { body } = await client.deleteByQuery({
|
||||
index: INDEX_ALIAS,
|
||||
body: params,
|
||||
conflicts: 'proceed',
|
||||
})
|
||||
|
||||
if (body.deleted > 0) {
|
||||
// * means deleting all pages of the same user
|
||||
await ctx.pubsub.entityDeleted(EntityType.PAGE, '*', ctx.uid)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
} catch (e) {
|
||||
logger.error('failed to delete pages by param in elastic', e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export const searchAsYouType = async (
|
||||
userId: string,
|
||||
query: string,
|
||||
size = 5
|
||||
): Promise<Page[]> => {
|
||||
try {
|
||||
const { body } = await client.search<SearchResponse<Page>>({
|
||||
index: INDEX_ALIAS,
|
||||
body: {
|
||||
query: {
|
||||
bool: {
|
||||
filter: [
|
||||
{
|
||||
term: {
|
||||
userId,
|
||||
},
|
||||
},
|
||||
{
|
||||
term: {
|
||||
state: ArticleSavingRequestStatus.Succeeded,
|
||||
},
|
||||
},
|
||||
{
|
||||
multi_match: {
|
||||
query,
|
||||
type: 'bool_prefix',
|
||||
fields: [
|
||||
'title',
|
||||
'title._2gram',
|
||||
'title._3gram',
|
||||
'siteName',
|
||||
'siteName._2gram',
|
||||
'siteName._3gram',
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
_source: ['title', 'slug', 'siteName', 'pageType'],
|
||||
size,
|
||||
},
|
||||
})
|
||||
|
||||
if (body.hits.total.value === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
return body.hits.hits.map((hit: { _source: Page; _id: string }) => ({
|
||||
...hit._source,
|
||||
id: hit._id,
|
||||
}))
|
||||
} catch (e) {
|
||||
logger.error('failed to search as you type in elastic', e)
|
||||
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
export const updatePages = async (
|
||||
ctx: PageContext,
|
||||
action: BulkActionType,
|
||||
args: PageSearchArgs,
|
||||
maxDocs: number,
|
||||
async: boolean,
|
||||
labels?: Label[]
|
||||
): Promise<string | null> => {
|
||||
// build the script
|
||||
let script = {
|
||||
source: '',
|
||||
params: {},
|
||||
}
|
||||
switch (action) {
|
||||
case BulkActionType.Archive:
|
||||
script = {
|
||||
source: `ctx._source.archivedAt = params.archivedAt;`,
|
||||
params: {
|
||||
archivedAt: new Date(),
|
||||
},
|
||||
}
|
||||
break
|
||||
case BulkActionType.Delete:
|
||||
script = {
|
||||
source: `ctx._source.state = params.state;`,
|
||||
params: {
|
||||
state: ArticleSavingRequestStatus.Deleted,
|
||||
},
|
||||
}
|
||||
break
|
||||
case BulkActionType.AddLabels:
|
||||
script = {
|
||||
source: `if (ctx._source.labels == null) {
|
||||
ctx._source.labels = params.labels
|
||||
} else {
|
||||
for (label in params.labels) {
|
||||
if (!ctx._source.labels.any(l -> l.name == label.name)) {
|
||||
ctx._source.labels.add(label)
|
||||
}
|
||||
}
|
||||
}`,
|
||||
params: {
|
||||
labels,
|
||||
},
|
||||
}
|
||||
break
|
||||
case BulkActionType.MarkAsRead:
|
||||
script = {
|
||||
source: `ctx._source.readAt = params.readAt;
|
||||
ctx._source.readingProgressPercent = params.readingProgressPercent;`,
|
||||
params: {
|
||||
readAt: new Date(),
|
||||
readingProgressPercent: 100,
|
||||
},
|
||||
}
|
||||
break
|
||||
default:
|
||||
throw new Error('Invalid bulk action')
|
||||
}
|
||||
|
||||
// add updatedAt to the script
|
||||
const updatedScript = {
|
||||
source: `${script.source} ctx._source.updatedAt = params.updatedAt`,
|
||||
lang: 'painless',
|
||||
params: {
|
||||
...script.params,
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
}
|
||||
|
||||
// build the query
|
||||
const searchBody = buildSearchBody(ctx.uid, args)
|
||||
.rawOption('script', updatedScript)
|
||||
.build()
|
||||
|
||||
logger.info('updating pages in elastic', searchBody)
|
||||
|
||||
try {
|
||||
const { body } = await client.updateByQuery({
|
||||
index: INDEX_ALIAS,
|
||||
conflicts: 'proceed',
|
||||
wait_for_completion: !async,
|
||||
body: searchBody,
|
||||
max_docs: maxDocs,
|
||||
requests_per_second: 500, // throttle the requests
|
||||
slices: 'auto', // parallelize the requests
|
||||
refresh: ctx.refresh,
|
||||
})
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
|
||||
if (body.failures && body.failures.length > 0) {
|
||||
logger.info('failed to update pages in elastic', body.failures)
|
||||
return null
|
||||
}
|
||||
|
||||
// TODO: publish entityUpdated events for each page
|
||||
|
||||
if (async) {
|
||||
logger.info('update pages task started', body.task)
|
||||
return body.task as string
|
||||
}
|
||||
|
||||
logger.info('updated pages in elastic', body.updated)
|
||||
return body.updated as string
|
||||
} catch (e) {
|
||||
logger.info('failed to update pages in elastic', e)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
import { logger } from '.'
|
||||
import { createPage, getPageByParam, updatePage } from './pages'
|
||||
import {
|
||||
ArticleSavingRequestStatus,
|
||||
Page,
|
||||
PageContext,
|
||||
Recommendation,
|
||||
} from './types'
|
||||
|
||||
export const addRecommendation = async (
|
||||
ctx: PageContext,
|
||||
page: Page,
|
||||
recommendation: Recommendation,
|
||||
highlightIds?: string[]
|
||||
): Promise<string | undefined> => {
|
||||
try {
|
||||
const highlights = page.highlights?.filter((highlight) =>
|
||||
highlightIds?.includes(highlight.id)
|
||||
)
|
||||
|
||||
// check if the page is already recommended to the group
|
||||
const existingPage = await getPageByParam({
|
||||
userId: ctx.uid,
|
||||
url: page.url,
|
||||
})
|
||||
if (existingPage) {
|
||||
const existingHighlights = existingPage.highlights || []
|
||||
|
||||
// remove duplicates
|
||||
const newHighlights =
|
||||
highlights?.filter(
|
||||
(highlight) =>
|
||||
!existingHighlights.find(
|
||||
(existingHighlight) => existingHighlight.quote === highlight.quote
|
||||
)
|
||||
) || []
|
||||
|
||||
const existingRecommendations = existingPage.recommendations || []
|
||||
const isRecommended = existingRecommendations.some(
|
||||
(existingRecommendation) =>
|
||||
existingRecommendation.id === recommendation.id
|
||||
)
|
||||
if (isRecommended && newHighlights.length === 0) {
|
||||
return existingPage.id
|
||||
}
|
||||
|
||||
// update recommendations in the existing page
|
||||
const recommendations = isRecommended
|
||||
? undefined
|
||||
: existingRecommendations.concat(recommendation)
|
||||
|
||||
await updatePage(
|
||||
existingPage.id,
|
||||
{
|
||||
recommendations,
|
||||
highlights: existingHighlights.concat(newHighlights),
|
||||
},
|
||||
ctx
|
||||
)
|
||||
|
||||
return existingPage.id
|
||||
}
|
||||
|
||||
// create a new page
|
||||
const newPage: Page = {
|
||||
...page,
|
||||
id: '',
|
||||
recommendations: [recommendation],
|
||||
userId: ctx.uid,
|
||||
readingProgressPercent: 0,
|
||||
readingProgressAnchorIndex: 0,
|
||||
sharedAt: new Date(),
|
||||
highlights,
|
||||
readAt: undefined,
|
||||
labels: undefined,
|
||||
subscription: undefined,
|
||||
unsubHttpUrl: undefined,
|
||||
unsubMailTo: undefined,
|
||||
_id: undefined,
|
||||
archivedAt: undefined,
|
||||
state: ArticleSavingRequestStatus.Succeeded,
|
||||
taskName: undefined,
|
||||
}
|
||||
|
||||
return createPage(newPage, ctx)
|
||||
} catch (err) {
|
||||
logger.error(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -85,7 +85,7 @@ export const revokeApiKeyResolver = authorized<
|
|||
RevokeApiKeySuccess,
|
||||
RevokeApiKeyError,
|
||||
MutationRevokeApiKeyArgs
|
||||
>(async (_, { id }, { claims: { uid }, log, authTrx }) => {
|
||||
>(async (_, { id }, { claims: { uid }, log }) => {
|
||||
try {
|
||||
const apiRepo = getRepository(ApiKey)
|
||||
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ import {
|
|||
findLibraryItemsByPrefix,
|
||||
searchLibraryItems,
|
||||
updateLibraryItem,
|
||||
updateLibraryItems,
|
||||
} from '../../services/library_item'
|
||||
import { parsedContentToLibraryItem } from '../../services/save_page'
|
||||
import {
|
||||
|
|
@ -832,21 +833,17 @@ export const bulkActionResolver = authorized<
|
|||
// parse query
|
||||
const searchQuery = parseSearchQuery(query)
|
||||
|
||||
const updated = await updateLibraryItem(action, searchQuery, labels)
|
||||
await updateLibraryItems(action, searchQuery, labels)
|
||||
|
||||
return { success: updated }
|
||||
return { success: true }
|
||||
} catch (error) {
|
||||
log.error('bulkActionResolver error', error)
|
||||
return { errorCodes: [BulkActionErrorCode.BadRequest] }
|
||||
}
|
||||
})
|
||||
|
||||
export type SetFavoriteArticleSuccessPartial = Merge<
|
||||
SetFavoriteArticleSuccess,
|
||||
{ favoriteArticle: PartialArticle }
|
||||
>
|
||||
export const setFavoriteArticleResolver = authorized<
|
||||
SetFavoriteArticleSuccessPartial,
|
||||
SetFavoriteArticleSuccess,
|
||||
SetFavoriteArticleError,
|
||||
MutationSetFavoriteArticleArgs
|
||||
>(async (_, { id }, { uid, log }) => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { updatePage } from '../../elastic/pages'
|
||||
import { LibraryItemState } from '../../entity/library_item'
|
||||
import { env } from '../../env'
|
||||
import {
|
||||
ArchiveLinkError,
|
||||
|
|
@ -6,6 +6,7 @@ import {
|
|||
ArchiveLinkSuccess,
|
||||
MutationSetLinkArchivedArgs,
|
||||
} from '../../generated/graphql'
|
||||
import { updateLibraryItem } from '../../services/library_item'
|
||||
import { analytics } from '../../utils/analytics'
|
||||
import { authorized } from '../../utils/helpers'
|
||||
|
||||
|
|
@ -52,11 +53,9 @@ export const setLinkArchivedResolver = authorized<
|
|||
ArchiveLinkSuccess,
|
||||
ArchiveLinkError,
|
||||
MutationSetLinkArchivedArgs
|
||||
>(async (_obj, args, { claims, pubsub, log }) => {
|
||||
log.info('setLinkArchivedResolver', args.input.linkId)
|
||||
|
||||
>(async (_obj, args, { uid }) => {
|
||||
analytics.track({
|
||||
userId: claims.uid,
|
||||
userId: uid,
|
||||
event: args.input.archived ? 'link_archived' : 'link_unarchived',
|
||||
properties: {
|
||||
env: env.server.apiEnv,
|
||||
|
|
@ -64,12 +63,15 @@ export const setLinkArchivedResolver = authorized<
|
|||
})
|
||||
|
||||
try {
|
||||
await updatePage(
|
||||
await updateLibraryItem(
|
||||
args.input.linkId,
|
||||
{
|
||||
archivedAt: args.input.archived ? new Date() : null,
|
||||
state: args.input.archived
|
||||
? LibraryItemState.Archived
|
||||
: LibraryItemState.Succeeded,
|
||||
},
|
||||
{ pubsub, uid: claims.uid, refresh: true } // refresh index to update search results
|
||||
uid
|
||||
)
|
||||
} catch (e) {
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import * as jwt from 'jsonwebtoken'
|
||||
import { RegistrationType } from '../../datalayer/user/model'
|
||||
import { appDataSource } from '../../data_source'
|
||||
import { deletePagesByParam } from '../../elastic/pages'
|
||||
import { User as UserEntity } from '../../entity/user'
|
||||
import { env } from '../../env'
|
||||
import {
|
||||
|
|
@ -38,7 +36,6 @@ import {
|
|||
UsersError,
|
||||
UsersSuccess,
|
||||
} from '../../generated/graphql'
|
||||
import { setClaims } from '../../repository'
|
||||
import { userRepository } from '../../repository/user'
|
||||
import { createUser } from '../../services/create_user'
|
||||
import { sendVerificationEmail } from '../../services/send_emails'
|
||||
|
|
@ -317,34 +314,9 @@ export const deleteAccountResolver = authorized<
|
|||
DeleteAccountSuccess,
|
||||
DeleteAccountError,
|
||||
MutationDeleteAccountArgs
|
||||
>(async (_, { userID }, { claims, log, pubsub }) => {
|
||||
const user = await userRepository.findOneBy({
|
||||
id: userID,
|
||||
})
|
||||
if (!user) {
|
||||
return {
|
||||
errorCodes: [DeleteAccountErrorCode.UserNotFound],
|
||||
}
|
||||
}
|
||||
|
||||
if (user.id !== claims.uid) {
|
||||
return {
|
||||
errorCodes: [DeleteAccountErrorCode.Unauthorized],
|
||||
}
|
||||
}
|
||||
|
||||
log.info('Deleting a user account', {
|
||||
userID,
|
||||
labels: {
|
||||
source: 'resolver',
|
||||
resolver: 'deleteAccountResolver',
|
||||
uid: claims.uid,
|
||||
},
|
||||
})
|
||||
|
||||
const result = await appDataSource.transaction(async (t) => {
|
||||
await setClaims(t, claims.uid)
|
||||
return t.getRepository(UserEntity).delete(userID)
|
||||
>(async (_, { userID }, { authTrx, log }) => {
|
||||
const result = await authTrx(async (t) => {
|
||||
return t.withRepository(userRepository).delete(userID)
|
||||
})
|
||||
if (!result.affected) {
|
||||
log.error('Error deleting user account')
|
||||
|
|
@ -354,9 +326,6 @@ export const deleteAccountResolver = authorized<
|
|||
}
|
||||
}
|
||||
|
||||
// delete this user's pages in elastic
|
||||
await deletePagesByParam({ userId: userID }, { uid: userID, pubsub })
|
||||
|
||||
return { userID }
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -4,18 +4,15 @@ import { htmlToSpeechFile } from '@omnivore/text-to-speech-handler'
|
|||
import cors from 'cors'
|
||||
import express from 'express'
|
||||
import * as jwt from 'jsonwebtoken'
|
||||
import { getPageById, updatePage } from '../elastic/pages'
|
||||
import { Speech, SpeechState } from '../entity/speech'
|
||||
import { Speech } from '../entity/speech'
|
||||
import { env } from '../env'
|
||||
import { CreateArticleErrorCode } from '../generated/graphql'
|
||||
import { createPubSubClient } from '../pubsub'
|
||||
import { getRepository } from '../repository'
|
||||
import { Claims } from '../resolvers/types'
|
||||
import { createPageSaveRequest } from '../services/create_page_save_request'
|
||||
import { findLibraryItemById } from '../services/library_item'
|
||||
import { getClaimsByToken } from '../utils/auth'
|
||||
import { isSiteBlockedForParse } from '../utils/blocked'
|
||||
import { corsConfig } from '../utils/corsConfig'
|
||||
import { enqueueTextToSpeech } from '../utils/createTask'
|
||||
import { logger } from '../utils/logger'
|
||||
import { generateDownloadSignedUrl } from '../utils/uploads'
|
||||
|
||||
|
|
@ -80,8 +77,7 @@ export function articleRouter() {
|
|||
async (req, res) => {
|
||||
const articleId = req.params.id
|
||||
const outputFormat = req.params.outputFormat
|
||||
const { voice, priority, secondaryVoice, language } =
|
||||
req.query as SpeechInput
|
||||
const { voice, secondaryVoice, language } = req.query as SpeechInput
|
||||
if (!articleId || outputFormats.indexOf(outputFormat) === -1) {
|
||||
return res.status(400).send('Invalid data')
|
||||
}
|
||||
|
|
@ -102,91 +98,21 @@ export function articleRouter() {
|
|||
})
|
||||
|
||||
try {
|
||||
if (outputFormat === 'speech') {
|
||||
const page = await getPageById(articleId)
|
||||
if (!page) {
|
||||
return res.status(404).send('Page not found')
|
||||
}
|
||||
if (page.userId !== uid) {
|
||||
logger.info('User is not allowed to access speech of the article', {
|
||||
userId: uid,
|
||||
articleId,
|
||||
})
|
||||
return res.status(401).send({ errorCode: 'UNAUTHORIZED' })
|
||||
}
|
||||
const speechFile = htmlToSpeechFile({
|
||||
title: page.title,
|
||||
content: page.content,
|
||||
options: {
|
||||
primaryVoice: voice,
|
||||
secondaryVoice: secondaryVoice,
|
||||
language: language || page.language,
|
||||
},
|
||||
})
|
||||
return res.send({ ...speechFile, pageId: articleId })
|
||||
}
|
||||
|
||||
const existingSpeech = await getRepository(Speech).findOne({
|
||||
where: {
|
||||
elasticPageId: articleId,
|
||||
voice,
|
||||
},
|
||||
order: {
|
||||
createdAt: 'DESC',
|
||||
},
|
||||
relations: ['user'],
|
||||
})
|
||||
if (existingSpeech) {
|
||||
if (existingSpeech.user.id !== uid) {
|
||||
logger.info('User is not allowed to access speech of the article', {
|
||||
userId: uid,
|
||||
articleId,
|
||||
})
|
||||
return res.status(401).send({ errorCode: 'UNAUTHORIZED' })
|
||||
}
|
||||
if (existingSpeech.state === SpeechState.COMPLETED) {
|
||||
logger.info('Found existing completed speech', {
|
||||
audioUrl: existingSpeech.audioFileName,
|
||||
speechMarksUrl: existingSpeech.speechMarksFileName,
|
||||
})
|
||||
await updatePage(
|
||||
existingSpeech.elasticPageId,
|
||||
{
|
||||
listenedAt: new Date(),
|
||||
},
|
||||
{ uid, pubsub: createPubSubClient() }
|
||||
)
|
||||
return res.redirect(await redirectUrl(existingSpeech, outputFormat))
|
||||
}
|
||||
if (existingSpeech.state === SpeechState.INITIALIZED) {
|
||||
logger.info('Found existing in progress speech')
|
||||
// retry later
|
||||
return res.status(202).send('Speech is in progress')
|
||||
}
|
||||
}
|
||||
|
||||
logger.info('Create Text to speech task', { articleId })
|
||||
const page = await getPageById(articleId)
|
||||
if (!page) {
|
||||
const item = await findLibraryItemById(articleId, uid)
|
||||
if (!item) {
|
||||
return res.status(404).send('Page not found')
|
||||
}
|
||||
// initialize state
|
||||
const speech = await getRepository(Speech).save({
|
||||
user: { id: uid },
|
||||
elasticPageId: articleId,
|
||||
state: SpeechState.INITIALIZED,
|
||||
voice,
|
||||
|
||||
const speechFile = htmlToSpeechFile({
|
||||
title: item.title,
|
||||
content: item.readableContent,
|
||||
options: {
|
||||
primaryVoice: voice,
|
||||
secondaryVoice: secondaryVoice,
|
||||
language: language || item.itemLanguage || undefined,
|
||||
},
|
||||
})
|
||||
// enqueue a task to convert text to speech
|
||||
const taskName = await enqueueTextToSpeech({
|
||||
userId: uid,
|
||||
speechId: speech.id,
|
||||
text: page.content,
|
||||
voice: speech.voice,
|
||||
priority: priority || 'high',
|
||||
})
|
||||
logger.info('Start Text to speech task', { taskName })
|
||||
res.status(202).send('Text to speech task started')
|
||||
return res.send({ ...speechFile, pageId: articleId })
|
||||
} catch (error) {
|
||||
logger.error('Error getting article speech:', error)
|
||||
res.status(500).send({ errorCode: 'INTERNAL_ERROR' })
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import {
|
|||
findLibraryItemByUrl,
|
||||
updateLibraryItem,
|
||||
} from '../services/library_item'
|
||||
import { addRecommendation } from '../services/recommendation'
|
||||
import { getTokenByRequest } from '../utils/auth'
|
||||
import { corsConfig } from '../utils/corsConfig'
|
||||
import {
|
||||
|
|
@ -166,18 +167,18 @@ export function pageRouter() {
|
|||
return res.status(404).send({ errorCode: 'NOT_FOUND' })
|
||||
}
|
||||
|
||||
const recommendedPageId = await addRecommendation(
|
||||
ctx,
|
||||
page,
|
||||
const recommendedItem = await addRecommendation(
|
||||
item,
|
||||
recommendation,
|
||||
claims.uid,
|
||||
highlightIds
|
||||
)
|
||||
if (!recommendedPageId) {
|
||||
if (!recommendedItem) {
|
||||
logger.error('Failed to add recommendation to page')
|
||||
return res.sendStatus(500)
|
||||
}
|
||||
|
||||
return res.send({ recommendedPageId })
|
||||
return res.send('OK')
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { DeepPartial, SelectQueryBuilder } from 'typeorm'
|
||||
import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity'
|
||||
import { Highlight } from '../entity/highlight'
|
||||
import { Label } from '../entity/label'
|
||||
import {
|
||||
|
|
@ -6,6 +7,7 @@ import {
|
|||
LibraryItemState,
|
||||
LibraryItemType,
|
||||
} from '../entity/library_item'
|
||||
import { BulkActionType } from '../generated/graphql'
|
||||
import { createPubSubClient, EntityType } from '../pubsub'
|
||||
import { authTrx } from '../repository'
|
||||
import { libraryItemRepository } from '../repository/library_item'
|
||||
|
|
@ -374,3 +376,47 @@ export const countByCreatedAt = async (
|
|||
.getCount()
|
||||
)
|
||||
}
|
||||
|
||||
export const updateLibraryItems = async (
|
||||
action: BulkActionType,
|
||||
args: SearchArgs,
|
||||
labels?: Label[]
|
||||
) => {
|
||||
// build the script
|
||||
let values: QueryDeepPartialEntity<LibraryItem> = {}
|
||||
switch (action) {
|
||||
case BulkActionType.Archive:
|
||||
values = {
|
||||
archivedAt: new Date(),
|
||||
}
|
||||
break
|
||||
case BulkActionType.Delete:
|
||||
values = {
|
||||
state: LibraryItemState.Deleted,
|
||||
}
|
||||
break
|
||||
case BulkActionType.AddLabels:
|
||||
values = {
|
||||
labels,
|
||||
}
|
||||
break
|
||||
case BulkActionType.MarkAsRead:
|
||||
values = {
|
||||
readAt: new Date(),
|
||||
readingProgressTopPercent: 100,
|
||||
readingProgressBottomPercent: 100,
|
||||
}
|
||||
break
|
||||
default:
|
||||
throw new Error('Invalid bulk action')
|
||||
}
|
||||
|
||||
await authTrx(async (tx) => {
|
||||
const queryBuilder = tx.createQueryBuilder(LibraryItem, 'library_item')
|
||||
|
||||
// build the where clause
|
||||
buildWhereClause(queryBuilder, args)
|
||||
|
||||
return queryBuilder.update(LibraryItem).set(values).execute()
|
||||
})
|
||||
}
|
||||
|
|
|
|||
82
packages/api/src/services/recommendation.ts
Normal file
82
packages/api/src/services/recommendation.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { DeepPartial } from 'typeorm'
|
||||
import { LibraryItem, LibraryItemState } from '../entity/library_item'
|
||||
import { Recommendation } from '../entity/recommendation'
|
||||
import { logger } from '../utils/logger'
|
||||
import {
|
||||
createLibraryItem,
|
||||
findLibraryItemByUrl,
|
||||
updateLibraryItem,
|
||||
} from './library_item'
|
||||
|
||||
export const addRecommendation = async (
|
||||
item: LibraryItem,
|
||||
recommendation: Recommendation,
|
||||
userId: string,
|
||||
highlightIds?: string[]
|
||||
) => {
|
||||
try {
|
||||
const highlights = item.highlights?.filter((highlight) =>
|
||||
highlightIds?.includes(highlight.id)
|
||||
)
|
||||
|
||||
// check if the item is already recommended to the group
|
||||
const existingItem = await findLibraryItemByUrl(item.originalUrl, userId)
|
||||
if (existingItem) {
|
||||
const existingHighlights = existingItem.highlights || []
|
||||
|
||||
// remove duplicates
|
||||
const newHighlights =
|
||||
highlights?.filter(
|
||||
(highlight) =>
|
||||
!existingHighlights.find(
|
||||
(existingHighlight) => existingHighlight.quote === highlight.quote
|
||||
)
|
||||
) || []
|
||||
|
||||
const existingRecommendations = existingItem.recommendations || []
|
||||
const isRecommended = existingRecommendations.some(
|
||||
(existingRecommendation) =>
|
||||
existingRecommendation.id === recommendation.id
|
||||
)
|
||||
if (isRecommended && newHighlights.length === 0) {
|
||||
return existingItem
|
||||
}
|
||||
|
||||
// update recommendations in the existing item
|
||||
const recommendations = isRecommended
|
||||
? undefined
|
||||
: existingRecommendations.concat(recommendation)
|
||||
|
||||
await updateLibraryItem(
|
||||
existingItem.id,
|
||||
{
|
||||
recommendations,
|
||||
highlights: existingHighlights.concat(newHighlights),
|
||||
},
|
||||
userId
|
||||
)
|
||||
|
||||
return existingItem
|
||||
}
|
||||
|
||||
// create a new item
|
||||
const newItem: DeepPartial<LibraryItem> = {
|
||||
...item,
|
||||
id: '',
|
||||
recommendations: [recommendation],
|
||||
user: { id: userId },
|
||||
readingProgressTopPercent: 0,
|
||||
readingProgressBottomPercent: 0,
|
||||
highlights,
|
||||
readAt: null,
|
||||
labels: [],
|
||||
archivedAt: null,
|
||||
state: LibraryItemState.Succeeded,
|
||||
}
|
||||
|
||||
return createLibraryItem(newItem, userId)
|
||||
} catch (err) {
|
||||
logger.error('Error adding recommendation', err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
@ -2,7 +2,7 @@ import axios from 'axios'
|
|||
import { NewsletterEmail } from '../entity/newsletter_email'
|
||||
import { Subscription } from '../entity/subscription'
|
||||
import { SubscriptionStatus, SubscriptionType } from '../generated/graphql'
|
||||
import { authTrx } from '../repository'
|
||||
import { authTrx, entityManager, getRepository } from '../repository'
|
||||
import { logger } from '../utils/logger'
|
||||
import { sendEmail } from '../utils/sendEmail'
|
||||
|
||||
|
|
@ -80,14 +80,13 @@ const sendUnsubscribeHttpRequest = async (url: string): Promise<boolean> => {
|
|||
}
|
||||
|
||||
export const getSubscriptionByName = async (
|
||||
name: string
|
||||
name: string,
|
||||
userId: string
|
||||
): Promise<Subscription | null> => {
|
||||
return authTrx((tx) =>
|
||||
tx.getRepository(Subscription).findOneBy({
|
||||
name,
|
||||
type: SubscriptionType.Newsletter,
|
||||
})
|
||||
)
|
||||
return getRepository(Subscription).findOne({
|
||||
where: { name, type: SubscriptionType.Newsletter, user: { id: userId } },
|
||||
relations: ['newsletterEmail', 'user'],
|
||||
})
|
||||
}
|
||||
|
||||
export const saveSubscription = async ({
|
||||
|
|
@ -105,13 +104,16 @@ export const saveSubscription = async ({
|
|||
lastFetchedAt: new Date(),
|
||||
}
|
||||
|
||||
const existingSubscription = await getSubscriptionByName(name)
|
||||
const result = await authTrx(async (tx) => {
|
||||
const existingSubscription = await getSubscriptionByName(name, userId)
|
||||
const result = await entityManager.transaction(async (tx) => {
|
||||
if (existingSubscription) {
|
||||
// update subscription if already exists
|
||||
await tx
|
||||
.getRepository(Subscription)
|
||||
.update(existingSubscription.id, subscriptionData)
|
||||
.update(
|
||||
{ id: existingSubscription.id, user: { id: userId } },
|
||||
subscriptionData
|
||||
)
|
||||
|
||||
return existingSubscription
|
||||
}
|
||||
|
|
|
|||
76
yarn.lock
76
yarn.lock
|
|
@ -11356,16 +11356,16 @@ color@3.0.x:
|
|||
color-convert "^1.9.1"
|
||||
color-string "^1.5.2"
|
||||
|
||||
colorette@2.0.19, colorette@^2.0.10, colorette@^2.0.14, colorette@^2.0.16:
|
||||
version "2.0.19"
|
||||
resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798"
|
||||
integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==
|
||||
|
||||
colorette@^1.2.2, colorette@^1.3.0:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.4.0.tgz#5190fbb87276259a86ad700bff2c6d6faa3fca40"
|
||||
integrity sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==
|
||||
|
||||
colorette@^2.0.10, colorette@^2.0.14, colorette@^2.0.16:
|
||||
version "2.0.19"
|
||||
resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798"
|
||||
integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==
|
||||
|
||||
colorspace@1.1.x:
|
||||
version "1.1.2"
|
||||
resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.2.tgz#e0128950d082b86a2168580796a0aa5d6c68d8c5"
|
||||
|
|
@ -11434,11 +11434,6 @@ commander@^8.3.0:
|
|||
resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66"
|
||||
integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==
|
||||
|
||||
commander@^9.0.0, commander@^9.1.0:
|
||||
version "9.5.0"
|
||||
resolved "https://registry.yarnpkg.com/commander/-/commander-9.5.0.tgz#bc08d1eb5cedf7ccb797a96199d41c7bc3e60d30"
|
||||
integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==
|
||||
|
||||
common-path-prefix@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/common-path-prefix/-/common-path-prefix-3.0.0.tgz#7d007a7e07c58c4b4d5f433131a19141b29f11e0"
|
||||
|
|
@ -13712,11 +13707,6 @@ eslint@^8.6.0:
|
|||
text-table "^0.2.0"
|
||||
v8-compile-cache "^2.0.3"
|
||||
|
||||
esm@^3.2.25:
|
||||
version "3.2.25"
|
||||
resolved "https://registry.yarnpkg.com/esm/-/esm-3.2.25.tgz#342c18c29d56157688ba5ce31f8431fbb795cc10"
|
||||
integrity sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==
|
||||
|
||||
espree@^9.0.0:
|
||||
version "9.4.1"
|
||||
resolved "https://registry.yarnpkg.com/espree/-/espree-9.4.1.tgz#51d6092615567a2c2cff7833445e37c28c0065bd"
|
||||
|
|
@ -14992,11 +14982,6 @@ get-value@^2.0.3, get-value@^2.0.6:
|
|||
resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28"
|
||||
integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=
|
||||
|
||||
getopts@2.3.0:
|
||||
version "2.3.0"
|
||||
resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.3.0.tgz#71e5593284807e03e2427449d4f6712a268666f4"
|
||||
integrity sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==
|
||||
|
||||
getos@^3.2.1:
|
||||
version "3.2.1"
|
||||
resolved "https://registry.yarnpkg.com/getos/-/getos-3.2.1.tgz#0134d1f4e00eb46144c5a9c0ac4dc087cbb27dc5"
|
||||
|
|
@ -18236,33 +18221,6 @@ klona@^2.0.4:
|
|||
resolved "https://registry.yarnpkg.com/klona/-/klona-2.0.5.tgz#d166574d90076395d9963aa7a928fabb8d76afbc"
|
||||
integrity sha512-pJiBpiXMbt7dkzXe8Ghj/u4FfXOOa98fPW+bihOJ4SjnoijweJrNThJfd3ifXpXhREjpoF2mZVH1GfS9LV3kHQ==
|
||||
|
||||
knex-stringcase@^1.4.2:
|
||||
version "1.4.6"
|
||||
resolved "https://registry.yarnpkg.com/knex-stringcase/-/knex-stringcase-1.4.6.tgz#e10f9201bc34de66386cc08525532012ebfd9e44"
|
||||
integrity sha512-8YJafQWJqF0REx6OYKem4UBBY75RD+ZvENcWAufz/afc9MOBGjFyr70EPBy59y6LqoPzSmCdUV+qWO550B3lJg==
|
||||
dependencies:
|
||||
stringcase "^4.3.1"
|
||||
|
||||
knex@2.4.2:
|
||||
version "2.4.2"
|
||||
resolved "https://registry.yarnpkg.com/knex/-/knex-2.4.2.tgz#a34a289d38406dc19a0447a78eeaf2d16ebedd61"
|
||||
integrity sha512-tMI1M7a+xwHhPxjbl/H9K1kHX+VncEYcvCx5K00M16bWvpYPKAZd6QrCu68PtHAdIZNQPWZn0GVhqVBEthGWCg==
|
||||
dependencies:
|
||||
colorette "2.0.19"
|
||||
commander "^9.1.0"
|
||||
debug "4.3.4"
|
||||
escalade "^3.1.1"
|
||||
esm "^3.2.25"
|
||||
get-package-type "^0.1.0"
|
||||
getopts "2.3.0"
|
||||
interpret "^2.2.0"
|
||||
lodash "^4.17.21"
|
||||
pg-connection-string "2.5.0"
|
||||
rechoir "^0.8.0"
|
||||
resolve-from "^5.0.0"
|
||||
tarn "^3.0.2"
|
||||
tildify "2.0.0"
|
||||
|
||||
kuler@^2.0.0:
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3"
|
||||
|
|
@ -21903,7 +21861,7 @@ performance-now@^2.1.0:
|
|||
resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
|
||||
integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
|
||||
|
||||
pg-connection-string@2.5.0, pg-connection-string@^2.5.0:
|
||||
pg-connection-string@^2.5.0:
|
||||
version "2.5.0"
|
||||
resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.5.0.tgz#538cadd0f7e603fc09a12590f3b8a452c2c0cf34"
|
||||
integrity sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==
|
||||
|
|
@ -23812,13 +23770,6 @@ rechoir@^0.7.0:
|
|||
dependencies:
|
||||
resolve "^1.9.0"
|
||||
|
||||
rechoir@^0.8.0:
|
||||
version "0.8.0"
|
||||
resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.8.0.tgz#49f866e0d32146142da3ad8f0eff352b3215ff22"
|
||||
integrity sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==
|
||||
dependencies:
|
||||
resolve "^1.20.0"
|
||||
|
||||
redent@^3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f"
|
||||
|
|
@ -25589,11 +25540,6 @@ string_decoder@~1.1.1:
|
|||
dependencies:
|
||||
safe-buffer "~5.1.0"
|
||||
|
||||
stringcase@^4.3.1:
|
||||
version "4.3.1"
|
||||
resolved "https://registry.yarnpkg.com/stringcase/-/stringcase-4.3.1.tgz#54279c9dd7379ff50f4e47d50d036b2ac888d34b"
|
||||
integrity sha512-Ov7McNX1sFaEX9NWijD1hIOVDDhKdnFzN9tvoa1N8xgrclouhsO4kBPVrTPhjO/zP5mn1Ww03uZ2SThNMXS7zg==
|
||||
|
||||
strip-ansi@^3.0.0, strip-ansi@^3.0.1:
|
||||
version "3.0.1"
|
||||
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
|
||||
|
|
@ -25949,11 +25895,6 @@ tar@^6.0.2, tar@^6.1.0:
|
|||
mkdirp "^1.0.3"
|
||||
yallist "^4.0.0"
|
||||
|
||||
tarn@^3.0.2:
|
||||
version "3.0.2"
|
||||
resolved "https://registry.yarnpkg.com/tarn/-/tarn-3.0.2.tgz#73b6140fbb881b71559c4f8bfde3d9a4b3d27693"
|
||||
integrity sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==
|
||||
|
||||
teeny-request@^8.0.0:
|
||||
version "8.0.0"
|
||||
resolved "https://registry.yarnpkg.com/teeny-request/-/teeny-request-8.0.0.tgz#9614410ba70114fd28ba7bf5077dce3e2f02adf7"
|
||||
|
|
@ -26166,11 +26107,6 @@ thunky@^1.0.2:
|
|||
resolved "https://registry.yarnpkg.com/thunky/-/thunky-1.1.0.tgz#5abaf714a9405db0504732bbccd2cedd9ef9537d"
|
||||
integrity sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==
|
||||
|
||||
tildify@2.0.0:
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/tildify/-/tildify-2.0.0.tgz#f205f3674d677ce698b7067a99e949ce03b4754a"
|
||||
integrity sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==
|
||||
|
||||
timers-browserify@^2.0.4:
|
||||
version "2.0.12"
|
||||
resolved "https://registry.yarnpkg.com/timers-browserify/-/timers-browserify-2.0.12.tgz#44a45c11fbf407f34f97bccd1577c652361b00ee"
|
||||
|
|
|
|||
Loading…
Reference in a new issue