prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

replace database subscriber with db trigger

Browse source
This commit is contained in:
Hongbo Wu 2023-08-31 15:26:30 +08:00
parent 612151c151
commit 381e9bc173
19 changed files with 275 additions and 307 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
packages/api/src/events/highlight_subscriber.ts
View file

@ -1,103 +0,0 @@
import {
EntityManager,
EntitySubscriberInterface,
EventSubscriber,
InsertEvent,
ObjectLiteral,
RemoveEvent,
UpdateEvent,
} from 'typeorm'
import { Highlight } from '../entity/highlight'
import { Label } from '../entity/label'
import { LibraryItem } from '../entity/library_item'
import { createPubSubClient, EntityType } from '../pubsub'
@EventSubscriber()
export class HighlightSubscriber
implements EntitySubscriberInterface<Highlight>
{
private readonly pubsubClient = createPubSubClient()
async updateLibraryItem(manager: EntityManager, libraryItemId: string) {
// get all the highlights belonging to the library_item
const highlights = await manager.getRepository(Highlight).find({
where: { libraryItem: { id: libraryItemId } },
relations: {
labels: true,
},
})
const highlightLabels: string[] = []
const highlightAnnotations: string[] = []
// for each highlight, add the lowercased label names to highlight_labels
// and the annotation to highlight_annotations
highlights.forEach((highlight) => {
highlight.labels &&
highlightLabels.push(
...highlight.labels.map((label) => label.name.toLowerCase())
)
highlightAnnotations.push(highlight.annotation || '')
})
// update highlight_labels and highlight_annotations on library_item
await manager.update(LibraryItem, libraryItemId, {
highlightAnnotations,
highlightLabels,
})
}
listenTo() {
return Highlight
}
async afterInsert(event: InsertEvent<Highlight>): Promise<void> {
await this.updateLibraryItem(event.manager, event.entity.libraryItem.id)
await this.pubsubClient.entityCreated<Highlight>(
EntityType.HIGHLIGHT,
event.entity,
event.entity.libraryItem.user.id
)
}
async afterUpdate(event: UpdateEvent<Highlight>): Promise<void> {
if (event.entity) {
await this.updateLibraryItem(
event.manager,
event.databaseEntity.libraryItem.id
)
// publish update event
await this.pubsubClient.entityUpdated<ObjectLiteral>(
EntityType.HIGHLIGHT,
{ ...event.entity, libraryItem: event.databaseEntity.libraryItem },
event.databaseEntity.libraryItem.user.id
)
// publish label added event if a label was added
if (event.entity.labels) {
await this.pubsubClient.entityCreated<Label>(
EntityType.LABEL,
event.entity.labels,
event.databaseEntity.libraryItem.user.id
)
}
}
}
async afterRemove(event: RemoveEvent<Highlight>): Promise<void> {
if (event.entityId) {
await this.updateLibraryItem(
event.manager,
event.databaseEntity.libraryItem.id
)
await this.pubsubClient.entityDeleted(
EntityType.HIGHLIGHT,
event.entityId,
event.databaseEntity.libraryItem.user.id
)
}
}
}

83
packages/api/src/events/library_item_subscriber.ts
View file

@ -1,83 +0,0 @@
import {
EntitySubscriberInterface,
EventSubscriber,
InsertEvent,
ObjectLiteral,
UpdateEvent,
} from 'typeorm'
import { Label } from '../entity/label'
import { LibraryItem, LibraryItemState } from '../entity/library_item'
import { createPubSubClient, EntityType } from '../pubsub'
@EventSubscriber()
export class LibraryItemSubscriber
implements EntitySubscriberInterface<LibraryItem>
{
private readonly pubsubClient = createPubSubClient()
listenTo() {
return LibraryItem
}
async afterInsert(event: InsertEvent<LibraryItem>): Promise<void> {
// Only publish the event if the library item has been successfully created
if (event.entity.state === LibraryItemState.Succeeded) {
await this.pubsubClient.entityCreated<LibraryItem>(
EntityType.PAGE,
event.entity,
event.entity.user.id
)
}
}
async afterUpdate(event: UpdateEvent<LibraryItem>): Promise<void> {
if (event.entity) {
// publish delete event if the library item has been deleted
if (event.entity.state === LibraryItemState.Deleted) {
return this.pubsubClient.entityDeleted(
EntityType.PAGE,
event.databaseEntity.id,
event.databaseEntity.user.id
)
}
// publish create event if the library item has finished processing
if (
event.databaseEntity.state === LibraryItemState.Processing &&
event.entity.state === LibraryItemState.Succeeded
) {
return this.pubsubClient.entityCreated<LibraryItem>(
EntityType.PAGE,
{
...event.databaseEntity,
...event.entity,
},
event.databaseEntity.user.id
)
}
// publish update event for all other cases
await this.pubsubClient.entityUpdated<ObjectLiteral>(
EntityType.PAGE,
event.entity,
event.databaseEntity.user.id
)
// publish label added event if a label was added
if (event.entity.labels) {
const labels = event.entity.labels as Label[]
await event.manager
.getRepository(LibraryItem)
.update(event.databaseEntity.id, {
labelNames: labels.map((label) => label.name.toLowerCase()),
})
await this.pubsubClient.entityCreated<Label>(
EntityType.LABEL,
event.entity.labels,
event.databaseEntity.user.id
)
}
}
}
}

4
packages/api/src/repository/api_key.ts Normal file
View file

@ -0,0 +1,4 @@
import { entityManager } from '.'
import { ApiKey } from '../entity/api_key'
export const apiKeyRepository = entityManager.getRepository(ApiKey)

17
packages/api/src/repository/index.ts
View file

@ -8,7 +8,6 @@ import { GroupMembership } from '../entity/groups/group_membership'
import { Invite } from '../entity/groups/invite'
import { Highlight } from '../entity/highlight'
import { Integration } from '../entity/integration'
import { Label } from '../entity/label'
import { LibraryItem } from '../entity/library_item'
import { NewsletterEmail } from '../entity/newsletter_email'
import { Profile } from '../entity/profile'
@ -19,19 +18,19 @@ import { AbuseReport } from '../entity/reports/abuse_report'
import { ContentDisplayReport } from '../entity/reports/content_display_report'
import { Rule } from '../entity/rule'
import { Subscription } from '../entity/subscription'
import { UploadFile } from '../entity/upload_file'
import { UserDeviceToken } from '../entity/user_device_tokens'
import { UserPersonalization } from '../entity/user_personalization'
import { Webhook } from '../entity/webhook'
export const setClaims = async (
manager: EntityManager,
uid: string
): Promise<void> => {
const dbRole = 'omnivore_user'
return manager
.query('SELECT * from omnivore.set_claims($1, $2)', [uid, dbRole])
.then()
uid = '00000000-0000-0000-0000-000000000000',
dbRole = 'omnivore_user'
): Promise<unknown> => {
return manager.query('SELECT * from omnivore.set_claims($1, $2)', [
uid,
dbRole,
])
}
export const getRepository = <T>(entity: EntityTarget<T>): Repository<T> => {
@ -40,7 +39,6 @@ export const getRepository = <T>(entity: EntityTarget<T>): Repository<T> => {
export const entityManager = appDataSource.manager
export const uploadFileRepository = getRepository(UploadFile)
export const reminderRepository = getRepository(Reminder)
export const libraryItemRepository = getRepository(LibraryItem)
export const groupMembershipRepository = getRepository(GroupMembership)
@ -54,7 +52,6 @@ export const filterRepository = getRepository(Filter)
export const followerRepository = getRepository(Follower)
export const highlightRepository = getRepository(Highlight)
export const integrationRepository = getRepository(Integration)
export const labelRepository = getRepository(Label)
export const newsletterEmailRepository = getRepository(NewsletterEmail)
export const profileRepository = getRepository(Profile)
export const receivedEmailRepository = getRepository(ReceivedEmail)

88
packages/api/src/repository/label.ts Normal file
View file

@ -0,0 +1,88 @@
import { In } from 'typeorm'
import { entityManager } from '.'
import { Label } from '../entity/label'
import { generateRandomColor } from '../utils/helpers'
const INTERNAL_LABELS_WITH_COLOR = new Map<
string,
{ name: string; color: string }
>([
['favorites', { name: 'Favorites', color: '#FFD700' }],
['library', { name: 'Library', color: '#584C42' }],
['rss', { name: 'RSS', color: '#F26522' }],
['newsletter', { name: 'Newsletter', color: '#07D2D1' }],
])
export const getInternalLabelWithColor = (name: string) => {
return INTERNAL_LABELS_WITH_COLOR.get(name.toLowerCase())
}
const isLabelInternal = (name: string): boolean => {
return INTERNAL_LABELS_WITH_COLOR.has(name.toLowerCase())
}
const toPartialLabel = (
label: {
name: string
color?: string | null
description?: string | null
},
userId: string
) => {
return {
user: { id: userId },
name: label.name,
color: label.color || generateRandomColor(), // assign a random color if not provided
description: label.description,
internal: isLabelInternal(label.name),
}
}
export const labelRepository = entityManager.getRepository(Label).extend({
findById(id: string) {
return this.findOneBy({ id })
},
findByName(name: string) {
return this.createQueryBuilder()
.where('LOWER(name) = LOWER(:name)', { name }) // case insensitive
.getOne()
},
findByNames(names: string[]) {
return this.createQueryBuilder()
.where('LOWER(name) IN (:...names)', {
names: names.map((n) => n.toLowerCase()),
})
.getMany()
},
findByIds(labelIds: string[]) {
return this.find({
where: { id: In(labelIds) },
select: ['id', 'name', 'color', 'description', 'createdAt'],
})
},
createLabel(
label: {
name: string
color?: string | null
description?: string | null
},
userId: string
) {
return this.save(toPartialLabel(label, userId))
},
createLabels(
labels: {
name: string
color?: string | null
description?: string | null
}[],
userId: string
) {
return this.save(labels.map((l) => toPartialLabel(l, userId)))
},
})

4
packages/api/src/repository/upload_file.ts Normal file
View file

@ -0,0 +1,4 @@
import { entityManager } from '.'
import { UploadFile } from '../entity/upload_file'
export const uploadFileRepository = entityManager.getRepository(UploadFile)

15
packages/api/src/resolvers/api_key/index.ts
View file

@ -20,8 +20,8 @@ import { authorized } from '../../utils/helpers'
export const apiKeysResolver = authorized<ApiKeysSuccess, ApiKeysError>(
async (_, __, { log, authTrx }) => {
try {
const apiKeys = await authTrx<Promise<ApiKey[]>>(async (em) => {
return em.getRepository(ApiKey).find({
const apiKeys = await authTrx<Promise<ApiKey[]>>(async (tx) => {
return tx.find(ApiKey, {
select: ['id', 'name', 'scopes', 'expiresAt', 'createdAt', 'usedAt'],
order: {
usedAt: { direction: 'DESC', nulls: 'last' },
@ -51,8 +51,8 @@ export const generateApiKeyResolver = authorized<
try {
const exp = new Date(expiresAt)
const originalKey = generateApiKey()
const apiKeyCreated = await authTrx<Promise<ApiKey>>(async (em) => {
return em.getRepository(ApiKey).save({
const apiKeyCreated = await authTrx<Promise<ApiKey>>(async (tx) => {
return tx.save(ApiKey, {
user: { id: uid },
name,
key: hashApiKey(originalKey),
@ -89,14 +89,13 @@ export const revokeApiKeyResolver = authorized<
MutationRevokeApiKeyArgs
>(async (_, { id }, { claims: { uid }, log, authTrx }) => {
try {
const deletedApiKey = await authTrx<Promise<ApiKey | null>>(async (em) => {
const apiKeyRepository = em.getRepository(ApiKey)
const apiKey = await apiKeyRepository.findOneBy({ id })
const deletedApiKey = await authTrx<Promise<ApiKey | null>>(async (tx) => {
const apiKey = await tx.findOneBy(ApiKey, { id })
if (!apiKey) {
return null
}
return apiKeyRepository.remove(apiKey)
return tx.remove(ApiKey, apiKey)
})
if (!deletedApiKey) {

90
packages/api/src/resolvers/article/index.ts
View file

@ -5,7 +5,9 @@
/* eslint-disable @typescript-eslint/no-floating-promises */
import { Readability } from '@omnivore/readability'
import graphqlFields from 'graphql-fields'
import { Label } from '../../entity/label'
import { LibraryItemType } from '../../entity/library_item'
import { UploadFile } from '../../entity/upload_file'
import { env } from '../../env'
import {
Article,
@ -57,16 +59,15 @@ import {
UpdateReason,
UpdatesSinceError,
UpdatesSinceErrorCode,
UpdatesSinceSuccess,
UpdatesSinceSuccess
} from '../../generated/graphql'
import { uploadFileRepository } from '../../repository'
import { getLibraryItemByUrl } from '../../repository/library_item'
import { getUserById, userRepository } from '../../repository/user'
import { userRepository } from '../../repository/user'
import { createPageSaveRequest } from '../../services/create_page_save_request'
import {
addLabelToPage,
createLabels,
getLabelsByIds,
getLabelsAndCreateIfNotExist,
getLabelsByIds
} from '../../services/labels'
import { searchLibraryItems } from '../../services/library_item'
import { setFileUploadComplete } from '../../services/save_file'
@ -75,7 +76,6 @@ import { traceAs } from '../../tracing'
import { Merge } from '../../util'
import { analytics } from '../../utils/analytics'
import { isSiteBlockedForParse } from '../../utils/blocked'
import { ContentParseError } from '../../utils/errors'
import {
authorized,
cleanUrl,
@ -85,7 +85,7 @@ import {
pageError,
titleForFilePath,
userDataToUser,
validatedDate,
validatedDate
} from '../../utils/helpers'
import { createImageProxyUrl } from '../../utils/imageproxy'
import {
@ -93,13 +93,13 @@ import {
getDistillerResult,
htmlToMarkdown,
ParsedContentPuppeteer,
parsePreparedContent,
parsePreparedContent
} from '../../utils/parser'
import { parseSearchQuery, SortBy, SortOrder } from '../../utils/search'
import {
contentReaderForPage,
getStorageFileDetails,
makeStorageFilePublic,
makeStorageFilePublic
} from '../../utils/uploads'
import { WithDataSourcesContext } from '../types'
import { itemTypeForContentType } from '../upload_files'
@ -231,28 +231,21 @@ export const createArticleResolver = authorized<
// save state
const archivedAt =
state === ArticleSavingRequestStatus.Archived ? new Date() : null
// if (pageId) {
// const reminder = await getRepository(Reminder).findOneBy({
// articleSavingRequest: pageId,
// user: { id: uid },
// })
// if (reminder && reminder.archiveUntil) {
// archivedAt = new Date()
// }
// }
// add labels to page
const labels = inputLabels
? await createLabels(ctx, inputLabels)
: undefined
// save labels
let labels: Label[] | undefined = undefined
if (inputLabels) {
labels = await getLabelsAndCreateIfNotExist(inputLabels, uid)
}
if (uploadFileId) {
/* We do not trust the values from client, lookup upload file by querying
* with filtering on user ID and URL to verify client's uploadFileId is valid.
*/
const uploadFile = await uploadFileRepository.findOneBy({
id: uploadFileId,
user: { id: uid },
})
const uploadFile = await authTrx((tx) =>
tx.findOneBy(UploadFile, {
id: uploadFileId,
})
)
if (!uploadFile) {
return pageError(
{ errorCodes: [CreateArticleErrorCode.UploadFileMissing] },
@ -293,9 +286,8 @@ export const createArticleResolver = authorized<
return DUMMY_RESPONSE
}
const saveTime = new Date()
const slug = generateSlug(parsedContent?.title || croppedPathname)
const articleToSave = parsedContentToLibraryItem({
const libraryItemToSave = parsedContentToLibraryItem({
url,
title,
parsedContent,
@ -309,11 +301,10 @@ export const createArticleResolver = authorized<
uploadFileHash,
canonicalUrl,
uploadFileId,
saveTime,
})
log.info('New article saving', {
parsedArticle: Object.assign({}, articleToSave, {
parsedArticle: Object.assign({}, libraryItemToSave, {
content: undefined,
originalHtml: undefined,
}),
@ -341,17 +332,17 @@ export const createArticleResolver = authorized<
await makeStorageFilePublic(uploadFileData.id, uploadFileData.fileName)
}
// save page's state and labels
articleToSave.archivedAt = archivedAt
articleToSave.labels = labels
libraryItemToSave.archivedAt = archivedAt
libraryItemToSave.labels = labels
const existingLibraryItem = await getLibraryItemByUrl(
articleToSave.originalUrl!,
libraryItemToSave.originalUrl!,
uid
)
pageId = existingLibraryItem?.id || pageId
if (pageId || existingLibraryItem) {
// update existing page's state from processing to succeeded
const updated = await updatePage(pageId, articleToSave, {
const updated = await updatePage(pageId, libraryItemToSave, {
...ctx,
uid,
})
@ -367,7 +358,7 @@ export const createArticleResolver = authorized<
}
} else {
// create new page in elastic
const newPageId = await createPage(articleToSave, { ...ctx, uid })
const newPageId = await createPage(libraryItemToSave, { ...ctx, uid })
if (!newPageId) {
return pageError(
{
@ -377,39 +368,26 @@ export const createArticleResolver = authorized<
pageId
)
}
articleToSave.id = newPageId
libraryItemToSave.id = newPageId
}
log.info(
'page created in elastic',
articleToSave.id,
articleToSave.url,
articleToSave.slug,
articleToSave.title
libraryItemToSave.id,
libraryItemToSave.url,
libraryItemToSave.slug,
libraryItemToSave.title
)
const createdArticle: PartialArticle = {
...articleToSave,
isArchived: !!articleToSave.archivedAt,
...libraryItemToSave,
isArchived: !!libraryItemToSave.archivedAt,
}
return {
user,
created: false,
createdArticle: createdArticle,
}
} catch (error) {
if (
error instanceof ContentParseError &&
error.message === 'UNABLE_TO_PARSE'
) {
return pageError(
{ errorCodes: [CreateArticleErrorCode.UnableToParse] },
ctx,
pageId
)
}
throw error
}
}
} }
)
export type ArticleSuccessPartial = Merge<

6
packages/api/src/resolvers/labels/index.ts
View file

@ -41,8 +41,8 @@ import { createPubSubClient } from '../../pubsub'
import { getRepository, setClaims } from '../../repository'
import {
createLabel,
createLabels,
getLabelByName,
getLabelsAndCreateIfNotExist,
getLabelsByIds,
} from '../../services/labels'
import { analytics } from '../../utils/analytics'
@ -252,7 +252,7 @@ export const setLabelsResolver = authorized<
if (labels && labels.length > 0) {
// for new clients that send label names
// create labels if they don't exist
labelsSet = await createLabels(ctx, labels)
labelsSet = await getLabelsAndCreateIfNotExist(ctx, labels)
} else if (labelIds && labelIds.length > 0) {
// for old clients that send labelIds
labelsSet = await getLabelsByIds(uid, labelIds)
@ -419,7 +419,7 @@ export const setLabelsForHighlightResolver = authorized<
if (labels && labels.length > 0) {
// for new clients that send label names
// create labels if they don't exist
labelsSet = await createLabels(ctx, labels)
labelsSet = await getLabelsAndCreateIfNotExist(ctx, labels)
} else if (labelIds && labelIds.length > 0) {
// for old clients that send labelIds
labelsSet = await getLabelsByIds(uid, labelIds)

2
packages/api/src/services/create_page_save_request.ts
View file

@ -94,7 +94,7 @@ export const createPageSaveRequest = async ({
try {
validateUrl(url)
} catch (error) {
logger.info('invalid url', url, error)
logger.error('invalid url', { url, error })
return Promise.reject({
errorCode: CreateArticleSavingRequestErrorCode.BadData,
})

56
packages/api/src/services/labels.ts
View file

@ -6,7 +6,8 @@ import { Label } from '../entity/label'
import { Link } from '../entity/link'
import { User } from '../entity/user'
import { CreateLabelInput } from '../generated/graphql'
import { getRepository, labelRepository } from '../repository'
import { entityManager, getRepository, setClaims } from '../repository'
import { labelRepository } from '../repository/label'
import { generateRandomColor } from '../utils/helpers'
import { logger } from '../utils/logger'
@ -120,35 +121,30 @@ export const createLabel = async (
})
}
export const createLabels = async (
ctx: PageContext,
labels: CreateLabelInput[]
export const getLabelsAndCreateIfNotExist = async (
labels: CreateLabelInput[],
userId: string
): Promise<Label[]> => {
const labelEntities = await labelRepository
.createQueryBuilder()
.where({
user: { id: ctx.uid },
})
.andWhere('LOWER(name) IN (:...names)', {
names: labels.map((l) => l.name.toLowerCase()),
})
.getMany()
return entityManager.transaction(async (tx) => {
await setClaims(tx, userId)
const existingLabelsInLowerCase = labelEntities.map((l) =>
l.name.toLowerCase()
)
const newLabels = labels.filter(
(l) => !existingLabelsInLowerCase.includes(l.name.toLowerCase())
)
// create new labels
const newLabelEntities = await labelRepository.save(
newLabels.map((l) => ({
name: l.name,
description: l.description,
color: l.color || generateRandomColor(),
internal: isLabelInternal(l.name),
user: { id: ctx.uid },
}))
)
return [...labelEntities, ...newLabelEntities]
const labelRepo = tx.withRepository(labelRepository)
// find existing labels
const labelEntities = await labelRepo.findByNames(labels.map((l) => l.name))
const existingLabelsInLowerCase = labelEntities.map((l) =>
l.name.toLowerCase()
)
const newLabels = labels.filter(
(l) => !existingLabelsInLowerCase.includes(l.name.toLowerCase())
)
if (newLabels.length === 0) {
return labelEntities
}
// create new labels
const newLabelEntities = await labelRepo.createLabels(newLabels, userId)
return [...labelEntities, ...newLabelEntities]
})
}

4
packages/api/src/services/save_file.ts
View file

@ -12,7 +12,7 @@ import { entityManager, getRepository } from '../repository'
import { WithDataSourcesContext } from '../resolvers/types'
import { logger } from '../utils/logger'
import { getStorageFileDetails } from '../utils/uploads'
import { createLabels } from './labels'
import { getLabelsAndCreateIfNotExist } from './labels'
export const setFileUploadComplete = async (
id: string,
@ -55,7 +55,7 @@ export const saveFile = async (
input.state === ArticleSavingRequestStatus.Archived ? new Date() : null
// add labels to page
const labels = input.labels
? await createLabels(ctx, input.labels)
? await getLabelsAndCreateIfNotExist(ctx, input.labels)
: undefined
if (input.state || input.labels) {
const updated = await updatePage(

4
packages/api/src/services/save_page.ts
View file

@ -30,7 +30,7 @@ import {
import { logger } from '../utils/logger'
import { parsePreparedContent } from '../utils/parser'
import { createPageSaveRequest } from './create_page_save_request'
import { createLabels } from './labels'
import { getLabelsAndCreateIfNotExist } from './labels'
import { createLibraryItem } from './library_item'
// where we can use APIs to fetch their underlying content.
@ -99,7 +99,7 @@ export const savePage = async (
input.state === ArticleSavingRequestStatus.Archived ? new Date() : null
// add labels to page
const labels = input.labels
? await createLabels(ctx, input.labels)
? await getLabelsAndCreateIfNotExist(ctx, input.labels)
: undefined
const isImported = input.source === 'csv-importer'

4
packages/api/src/services/save_url.ts
View file

@ -6,7 +6,7 @@ import { PubsubClient } from '../pubsub'
import { getRepository } from '../repository'
import { logger } from '../utils/logger'
import { createPageSaveRequest } from './create_page_save_request'
import { createLabels } from './labels'
import { getLabelsAndCreateIfNotExist } from './labels'
interface SaveContext {
pubsub: PubsubClient
@ -24,7 +24,7 @@ export const saveUrl = async (
input.state === ArticleSavingRequestStatus.Archived ? new Date() : null
// add labels to page
const labels = input.labels
? await createLabels(ctx, input.labels)
? await getLabelsAndCreateIfNotExist(ctx, input.labels)
: undefined
const pageSaveRequest = await createPageSaveRequest({

10
packages/db/migrations/0118.do.library_item.sql
View file

@ -55,9 +55,9 @@ CREATE TABLE omnivore.library_item (
gcs_archive_id text,
directionality directionality_type NOT NULL DEFAULT 'LTR',
subscription_id uuid REFERENCES omnivore.subscriptions ON DELETE CASCADE,
label_names text[], -- array of label names of the item
highlight_labels text[], -- array of label names of the item's highlights
highlight_annotations text[], -- array of highlight annotations of the item
label_names text[] NOT NULL DEFAULT array[]::text[], -- array of label names of the item
highlight_labels text[] NOT NULL DEFAULT array[]::text[], -- array of label names of the item's highlights
highlight_annotations text[] NOT NULL DEFAULT array[]::text[], -- array of highlight annotations of the item
note text,
note_tsv tsvector,
UNIQUE (user_id, original_url)
@ -81,7 +81,7 @@ begin
new.author_tsv := to_tsvector('pg_catalog.english', coalesce(new.author, ''));
new.description_tsv := to_tsvector('pg_catalog.english', coalesce(new.description, ''));
-- note_tsv is generated by both note and highlight_annotations
new.note_tsv := to_tsvector('pg_catalog.english', coalesce(new.note, '')) || to_tsvector('pg_catalog.english', string_agg(new.highlight_annotations, ' '));
new.note_tsv := to_tsvector('pg_catalog.english', coalesce(new.note, '') || ' ' || array_to_string(new.highlight_annotations, ' '));
new.search_tsv :=
setweight(new.title_tsv, 'A') ||
setweight(new.author_tsv, 'A') ||
@ -91,7 +91,7 @@ begin
setweight(to_tsvector('pg_catalog.english', coalesce(regexp_replace(new.original_url, '^((http[s]?):\/)?\/?([^:\/\s]+)((\/\w+)*\/)([\w\-\.]+[^#?\s]+)(.*)?(#[\w\-]+)?$', '\3'), '')), 'A') ||
-- secondary hostname (eg omnivore)
setweight(to_tsvector('pg_catalog.english', coalesce(regexp_replace(new.original_url, '^((http[s]?):\/)?\/?(.*\.)?([^:\/\s]+)(\..*)((\/+)*\/)?([\w\-\.]+[^#?\s]+)(.*)?(#[\w\-]+)?$', '\4'), '')), 'A') ||
setweight(to_tsvector('pg_catalog.english', new.note_tsv), 'A') ||
setweight(new.note_tsv, 'A') ||
setweight(new.content_tsv, 'B');
return new;
end

56
packages/db/migrations/0119.do.entity_labels.sql
View file

@ -5,10 +5,62 @@
BEGIN;
CREATE TABLE omnivore.entity_labels (
id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
library_item_id uuid REFERENCES omnivore.library_item(id) ON DELETE CASCADE,
highlight_id uuid REFERENCES omnivore.highlight(id) ON DELETE CASCADE,
label_id uuid NOT NULL REFERENCES omnivore.labels(id) ON DELETE CASCADE,
PRIMARY KEY (library_item_id, highlight_id, label_id)
label_id uuid NOT NULL REFERENCES omnivore.labels(id) ON DELETE CASCADE
);
CREATE OR REPLACE FUNCTION update_library_item_labels()
RETURNS trigger AS $$
DECLARE
current_library_item_id uuid;
current_highlight_id uuid;
BEGIN
IF TG_OP = 'DELETE' THEN
current_library_item_id = OLD.library_item_id;
current_highlight_id = OLD.highlight_id;
ELSE
current_library_item_id = NEW.library_item_id;
current_highlight_id = NEW.highlight_id;
END IF;
IF current_library_item_id IS NOT NULL THEN
-- for labels of the ABORTlibrary_item
WITH labels_agg AS (
SELECT array_agg(l.name) as names_agg
FROM omnivore.labels l
INNER JOIN omnivore.entity_labels el ON el.label_id = l.id AND el.library_item_id = current_library_item_id
)
-- Update label_names on library_item
UPDATE omnivore.library_item li
SET label_names = l.names_agg
FROM labels_agg l
WHERE li.id = current_library_item_id;
ELSIF current_highlight_id IS NOT NULL THEN
-- for labels of highlights of the library item
current_library_item_id = (SELECT library_item_id FROM omnivore.highlight WHERE id = current_highlight_id);
WITH labels_agg AS (
SELECT array_agg(l.name) as names_agg
FROM omnivore.labels l
INNER JOIN omnivore.entity_labels el ON el.label_id = l.id
INNER JOIN omnivore.highlight h ON h.id = el.highlight_id AND h.library_item_id = current_library_item_id
)
-- Update highlight_labels on library_item
UPDATE omnivore.library_item li
SET highlight_labels = l.names_agg
FROM labels_agg l
WHERE li.id = current_library_item_id;
END IF;
return NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER library_item_labels_update
AFTER INSERT OR UPDATE OR DELETE ON omnivore.entity_labels
FOR EACH ROW
EXECUTE FUNCTION update_library_item_labels();
COMMIT;

3
packages/db/migrations/0119.undo.entity_labels.sql
View file

@ -4,6 +4,9 @@
BEGIN;
DROP TRIGGER library_item_labels_update ON omnivore.entity_labels;
DROP FUNCTION update_library_item_labels();
DROP TABLE omnivore.entity_labels;
COMMIT;

30
packages/db/migrations/0121.do.update_highlight.sql
View file

@ -20,4 +20,34 @@ ALTER TABLE omnivore.highlight
DROP COLUMN article_id,
DROP COLUMN elastic_page_id;
CREATE OR REPLACE FUNCTION update_library_item_highlight_annotations()
RETURNS TRIGGER AS $$
DECLARE
current_library_item_id uuid;
BEGIN
IF TG_OP = 'INSERT' OR TG_OP = 'UPDATE' THEN
current_library_item_id = NEW.library_item_id;
ELSE
current_library_item_id = OLD.library_item_id;
END IF;
WITH highlight_agg AS (
SELECT array_agg(coalesce(annotation, '')) AS annotation_agg
FROM omnivore.highlight
WHERE library_item_id = current_library_item_id
)
UPDATE omnivore.library_item li
SET highlight_annotations = h.annotation_agg
FROM highlight_agg h
WHERE li.id = current_library_item_id;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER library_item_highlight_annotations_update
AFTER INSERT OR UPDATE OR DELETE ON omnivore.highlight
FOR EACH ROW
EXECUTE FUNCTION update_library_item_highlight_annotations();
COMMIT;

3
packages/db/migrations/0121.undo.update_highlight.sql
View file

@ -4,6 +4,9 @@
BEGIN;
DROP TRIGGER IF EXISTS library_item_highlight_annotations_update ON omnivore.highlight;
DROP FUNCTION IF EXISTS update_library_item_highlight_annotations();
ALTER TABLE omnivore.highlight
ADD COLUMN article_id uuid,
ADD COLUMN elastic_page_id uuid,