diff --git a/packages/api/src/entity/highlight.ts b/packages/api/src/entity/highlight.ts index 8147fa2d5..73e196555 100644 --- a/packages/api/src/entity/highlight.ts +++ b/packages/api/src/entity/highlight.ts @@ -16,7 +16,6 @@ import { User } from './user' export enum HighlightType { Highlight = 'HIGHLIGHT', Redaction = 'REDACTION', // allowing people to remove text from the page - Note = 'NOTE', // allowing people to add a note at the document level } @Entity({ name: 'highlight' }) @@ -50,9 +49,6 @@ export class Highlight { @Column('text') annotation?: string | null - @Column('boolean') - deleted?: boolean - @CreateDateColumn() createdAt!: Date diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts index 4399100e5..5a86933a0 100644 --- a/packages/api/src/entity/library_item.ts +++ b/packages/api/src/entity/library_item.ts @@ -15,7 +15,6 @@ import { import { Highlight } from './highlight' import { Label } from './label' import { Recommendation } from './recommendation' -import { Subscription } from './subscription' import { UploadFile } from './upload_file' import { User } from './user' @@ -164,9 +163,8 @@ export class LibraryItem { @Column('text', { nullable: true }) gcsArchiveId?: string | null - @OneToOne(() => Subscription, { cascade: true }) - @JoinColumn({ name: 'subscription_id' }) - subscription?: Subscription + @Column('text', { nullable: true }) + subscription?: string | null @ManyToMany(() => Label, { cascade: true }) @JoinTable({ diff --git a/packages/api/src/entity/recommendation.ts b/packages/api/src/entity/recommendation.ts index d42c84cc5..e4007a515 100644 --- a/packages/api/src/entity/recommendation.ts +++ b/packages/api/src/entity/recommendation.ts @@ -6,6 +6,7 @@ import { ManyToOne, PrimaryGeneratedColumn, } from 'typeorm' +import { Group } from './groups/group' import { LibraryItem } from './library_item' import { User } from './user' @@ -22,6 +23,10 @@ export class Recommendation { @JoinColumn({ name: 'library_item_id' }) libraryItem!: LibraryItem + @ManyToOne(() => Group, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'group_id' }) + group!: Group + @Column('text', { nullable: true }) note?: string | null diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts index b72113048..a5cd8eed2 100644 --- a/packages/api/src/generated/graphql.ts +++ b/packages/api/src/generated/graphql.ts @@ -102,6 +102,7 @@ export type Article = { labels?: Maybe>; language?: Maybe; linkId?: Maybe; + note?: Maybe; originalArticleUrl?: Maybe; originalHtml?: Maybe; pageType?: Maybe; @@ -949,7 +950,6 @@ export type HighlightStats = { export enum HighlightType { Highlight = 'HIGHLIGHT', - Note = 'NOTE', Redaction = 'REDACTION' } @@ -2212,6 +2212,7 @@ export type SearchItem = { isArchived: Scalars['Boolean']; labels?: Maybe>; language?: Maybe; + note?: Maybe; originalArticleUrl?: Maybe; ownedByViewer?: Maybe; pageId?: Maybe; @@ -4292,6 +4293,7 @@ export type ArticleResolvers>, ParentType, ContextType>; language?: Resolver, ParentType, ContextType>; linkId?: Resolver, ParentType, ContextType>; + note?: Resolver, ParentType, ContextType>; originalArticleUrl?: Resolver, ParentType, ContextType>; originalHtml?: Resolver, ParentType, ContextType>; pageType?: Resolver, ParentType, ContextType>; @@ -5528,6 +5530,7 @@ export type SearchItemResolvers; labels?: Resolver>, ParentType, ContextType>; language?: Resolver, ParentType, ContextType>; + note?: Resolver, ParentType, ContextType>; originalArticleUrl?: Resolver, ParentType, ContextType>; ownedByViewer?: Resolver, ParentType, ContextType>; pageId?: Resolver, ParentType, ContextType>; diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql index b70838275..b53827017 100644 --- a/packages/api/src/generated/schema.graphql +++ b/packages/api/src/generated/schema.graphql @@ -78,6 +78,7 @@ type Article { labels: [Label!] language: String linkId: ID + note: String originalArticleUrl: String originalHtml: String pageType: PageType @@ -844,7 +845,6 @@ type HighlightStats { enum HighlightType { HIGHLIGHT - NOTE REDACTION } @@ -1684,6 +1684,7 @@ type SearchItem { isArchived: Boolean! labels: [Label!] language: String + note: String originalArticleUrl: String ownedByViewer: Boolean pageId: ID diff --git a/packages/api/src/repository/highlight.ts b/packages/api/src/repository/highlight.ts index e38da3e0b..b2eaab20f 100644 --- a/packages/api/src/repository/highlight.ts +++ b/packages/api/src/repository/highlight.ts @@ -6,10 +6,12 @@ import { unescapeHtml } from '../utils/helpers' const unescapeHighlight = (highlight: DeepPartial) => { // unescape HTML entities - highlight.annotation = highlight.annotation - ? unescapeHtml(highlight.annotation) - : undefined - highlight.quote = highlight.quote ? unescapeHtml(highlight.quote) : undefined + if (highlight.annotation !== undefined && highlight.annotation !== null) { + highlight.annotation = unescapeHtml(highlight.annotation.toString()) + } + if (highlight.quote !== undefined && highlight.quote !== null) { + highlight.quote = unescapeHtml(highlight.quote.toString()) + } return highlight } @@ -21,9 +23,10 @@ export const highlightRepository = entityManager return this.findOneBy({ id }) }, - findByLibraryItemId(libraryItemId: string) { + findByLibraryItemId(libraryItemId: string, userId: string) { return this.findBy({ libraryItem: { id: libraryItemId }, + user: { id: userId }, }) }, @@ -35,14 +38,12 @@ export const highlightRepository = entityManager highlightId: string, highlight: QueryDeepPartialEntity ) { - return this.update(highlightId, { - ...highlight, - annotation: highlight.annotation - ? unescapeHtml(highlight.annotation.toString()) - : undefined, - quote: highlight.quote - ? unescapeHtml(highlight.quote.toString()) - : undefined, - }) + if (highlight.annotation !== undefined && highlight.annotation !== null) { + highlight.annotation = unescapeHtml(highlight.annotation.toString()) + } + if (highlight.quote !== undefined && highlight.quote !== null) { + highlight.quote = unescapeHtml(highlight.quote.toString()) + } + return this.update(highlightId, highlight) }, }) diff --git a/packages/api/src/resolvers/article/index.ts b/packages/api/src/resolvers/article/index.ts index 67459d3b3..d66f4f65e 100644 --- a/packages/api/src/resolvers/article/index.ts +++ b/packages/api/src/resolvers/article/index.ts @@ -4,6 +4,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-floating-promises */ import { Readability } from '@omnivore/readability' +import graphqlFields from 'graphql-fields' import { DeepPartial } from 'typeorm' import { LibraryItem, @@ -377,9 +378,9 @@ export const getArticleResolver = authorized< QueryArticleArgs >(async (_obj, { slug, format }, { authTrx, uid, log }, info) => { try { - // const includeOriginalHtml = - // format === ArticleFormat.Distiller || - // !!graphqlFields(info).article.originalHtml + const includeOriginalHtml = + format === ArticleFormat.Distiller || + !!graphqlFields(info).article.originalHtml // We allow the backend to use the ID instead of a slug to fetch the article const libraryItem = await authTrx((tx) => @@ -392,6 +393,10 @@ export const getArticleResolver = authorized< labels: true, }, uploadFile: true, + recommendations: { + recommender: true, + group: true, + }, }, }) ) diff --git a/packages/api/src/resolvers/highlight/index.ts b/packages/api/src/resolvers/highlight/index.ts index e259613b2..61b49f5ba 100644 --- a/packages/api/src/resolvers/highlight/index.ts +++ b/packages/api/src/resolvers/highlight/index.ts @@ -88,7 +88,7 @@ export const mergeHighlightResolver = authorized< MergeHighlightSuccess, MergeHighlightError, MutationMergeHighlightArgs ->(async (_, { input }, { authTrx, log, pubsub, uid }) => { +>(async (_, { input }, { log, pubsub, uid }) => { const { overlapHighlightIdList, ...newHighlightInput } = input /* Compute merged annotation form the order of highlights appearing on page */ @@ -97,11 +97,10 @@ export const mergeHighlightResolver = authorized< const mergedColors: string[] = [] try { - const existingHighlights = await authTrx(async (tx) => { - return tx - .withRepository(highlightRepository) - .findByLibraryItemId(input.articleId) - }) + const existingHighlights = await highlightRepository.findByLibraryItemId( + input.articleId, + uid + ) existingHighlights.forEach((highlight) => { // filter out highlights that are in the overlap list @@ -178,9 +177,9 @@ export const updateHighlightResolver = authorized< const updatedHighlight = await updateHighlight( input.highlightId, { - annotation: input.annotation ?? undefined, - html: input.html ?? undefined, - quote: input.quote ?? undefined, + annotation: input.annotation, + html: input.html, + quote: input.quote, }, uid, pubsub diff --git a/packages/api/src/resolvers/recommendations/index.ts b/packages/api/src/resolvers/recommendations/index.ts index efe645271..cbfcbc9cf 100644 --- a/packages/api/src/resolvers/recommendations/index.ts +++ b/packages/api/src/resolvers/recommendations/index.ts @@ -208,7 +208,7 @@ export const recommendResolver = authorized< member.user.id, item.id, { - id: group.id, + group, note: input.note ?? null, recommender: item.user, createdAt: new Date(), @@ -227,14 +227,7 @@ export const recommendResolver = authorized< success: true, } } catch (error) { - log.error('Error recommending', { - error, - labels: { - source: 'resolver', - resolver: 'recommendResolver', - uid, - }, - }) + log.error('Error recommending', error) return { errorCodes: [RecommendErrorCode.BadRequest], diff --git a/packages/api/src/routers/page_router.ts b/packages/api/src/routers/page_router.ts index 2de3681ab..3db753c13 100644 --- a/packages/api/src/routers/page_router.ts +++ b/packages/api/src/routers/page_router.ts @@ -152,17 +152,17 @@ export function pageRouter() { } const claims = jwt.decode(token) as Claims - const { userId, pageId, recommendation, highlightIds } = req.body as { + const { userId, itemId, recommendation, highlightIds } = req.body as { userId: string - pageId: string + itemId: string recommendation: Recommendation highlightIds?: string[] } - if (!userId || !pageId || !recommendation) { + if (!userId || !itemId || !recommendation) { return res.status(400).send({ errorCode: 'BAD_DATA' }) } - const item = await findLibraryItemById(pageId, userId) + const item = await findLibraryItemById(itemId, userId) if (!item) { return res.status(404).send({ errorCode: 'NOT_FOUND' }) } diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 14e257fd3..3d279baa7 100755 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -394,6 +394,7 @@ const schema = gql` readAt: Date recommendations: [Recommendation!] wordsCount: Int + note: String } # Query: article @@ -686,7 +687,6 @@ const schema = gql` enum HighlightType { HIGHLIGHT REDACTION - NOTE } # Highlight @@ -1606,6 +1606,7 @@ const schema = gql` wordsCount: Int content: String archivedAt: Date + note: String } type SearchItemEdge { diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 792ea1a83..2aa9e1784 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -123,12 +123,12 @@ const buildWhereClause = ( break case InFilter.SUBSCRIPTION: queryBuilder - .andWhere('library_item.subscription_id IS NOT NULL') + .andWhere('library_item.subscription IS NOT NULL') .andWhere('library_item.archived_at IS NULL') break case InFilter.LIBRARY: queryBuilder - .andWhere('library_item.subscription_id IS NULL') + .andWhere('library_item.subscription IS NULL') .andWhere('library_item.archived_at IS NULL') break } @@ -248,19 +248,24 @@ const buildWhereClause = ( } if (args.recommendedBy) { - queryBuilder.innerJoin('library_item.recommendations', 'recommendations') + const recommendedByInLowerCase = args.recommendedBy.toLowerCase() + queryBuilder + .innerJoin('library_item.recommendations', 'recommendations') + .innerJoin('recommendations.recommender', 'recommender') + .innerJoin('recommendations.group', 'group') + .andWhere((qb) => { + qb.where('lower(recommender.name) = :recommendedBy', { + recommendedBy: recommendedByInLowerCase, + }).orWhere('lower(group.name) = :recommendedBy', { + recommendedBy: recommendedByInLowerCase, + }) + }) } if (args.subscription) { - queryBuilder - .innerJoin('library_item.subscription', 'subscription') - .andWhere((qb) => { - qb.where('subscription.name = :subscription', { - subscription: args.subscription, - }).orWhere('subscription.url = :subscription', { - subscription: args.subscription, - }) - }) + queryBuilder.andWhere('lower(library_item.subscription) = :subscription', { + subscription: args.subscription.toLowerCase(), + }) } } @@ -314,8 +319,7 @@ export const findLibraryItemById = async ( .createQueryBuilder(LibraryItem, 'library_item') .leftJoinAndSelect('library_item.labels', 'labels') .leftJoinAndSelect('library_item.highlights', 'highlights') - .where('library_item.user_id = :userId', { userId }) - .andWhere('library_item.id = :id', { id }) + .where('library_item.id = :id', { id }) .getOne(), undefined, userId @@ -332,8 +336,11 @@ export const findLibraryItemByUrl = async ( .createQueryBuilder(LibraryItem, 'library_item') .leftJoinAndSelect('library_item.labels', 'labels') .leftJoinAndSelect('library_item.highlights', 'highlights') - .where('library_item.user_id = :userId', { userId }) - .andWhere('library_item.original_url = :url', { url }) + .leftJoinAndSelect('library_item.recommendations', 'recommendations') + .leftJoinAndSelect('recommendations.recommender', 'recommender') + .leftJoinAndSelect('recommender.profile', 'profile') + .leftJoinAndSelect('recommendations.group', 'group') + .where('library_item.original_url = :url', { url }) .getOne(), undefined, userId @@ -416,11 +423,15 @@ export const findLibraryItemsByPrefix = async ( prefix: string, limit = 5 ): Promise => { + const prefixWildcard = `${prefix}%` + return authTrx(async (tx) => tx .createQueryBuilder(LibraryItem, 'library_item') - .where('library_item.title ILIKE :prefix', { prefix: `${prefix}%` }) - .orWhere('library_item.site_name ILIKE :prefix', { prefix: `${prefix}%` }) + .where('library_item.title ILIKE :prefix', { prefix: prefixWildcard }) + .orWhere('library_item.site_name ILIKE :prefix', { + prefix: prefixWildcard, + }) .orderBy('library_item.saved_at', 'DESC') .limit(limit) .getMany() diff --git a/packages/api/src/services/recommendation.ts b/packages/api/src/services/recommendation.ts index b65ef4c6c..dc6c85d9e 100644 --- a/packages/api/src/services/recommendation.ts +++ b/packages/api/src/services/recommendation.ts @@ -1,5 +1,5 @@ import { DeepPartial } from 'typeorm' -import { LibraryItem, LibraryItemState } from '../entity/library_item' +import { LibraryItem } from '../entity/library_item' import { Recommendation } from '../entity/recommendation' import { logger } from '../utils/logger' import { @@ -34,23 +34,12 @@ export const addRecommendation = async ( ) || [] const existingRecommendations = existingItem.recommendations || [] - const isRecommended = existingRecommendations.some( - (existingRecommendation) => - existingRecommendation.id === recommendation.id - ) - if (isRecommended && newHighlights.length === 0) { - return existingItem - } // update recommendations in the existing item - const recommendations = isRecommended - ? undefined - : existingRecommendations.concat(recommendation) - await updateLibraryItem( existingItem.id, { - recommendations, + recommendations: existingRecommendations.concat(recommendation), highlights: existingHighlights.concat(newHighlights), }, userId @@ -61,17 +50,25 @@ export const addRecommendation = async ( // create a new item const newItem: DeepPartial = { - ...item, - id: '', recommendations: [recommendation], user: { id: userId }, - readingProgressTopPercent: 0, - readingProgressBottomPercent: 0, highlights, - readAt: null, - labels: [], - archivedAt: null, - state: LibraryItemState.Succeeded, + slug: item.slug, + title: item.title, + author: item.author, + description: item.description, + originalUrl: item.originalUrl, + originalContent: item.originalContent, + contentReader: item.contentReader, + directionality: item.directionality, + itemLanguage: item.itemLanguage, + itemType: item.itemType, + readableContent: item.readableContent, + siteIcon: item.siteIcon, + siteName: item.siteName, + thumbnail: item.thumbnail, + uploadFile: item.uploadFile, + wordCount: item.wordCount, } return createLibraryItem(newItem, userId) diff --git a/packages/api/src/services/save_email.ts b/packages/api/src/services/save_email.ts index 1c5976023..a3d040c02 100644 --- a/packages/api/src/services/save_email.ts +++ b/packages/api/src/services/save_email.ts @@ -115,6 +115,7 @@ export const saveEmail = async ( siteIcon, siteName: parseResult.parsedContent?.siteName ?? undefined, wordCount: wordsCount(content), + subscription: input.author, }, input.userId ) diff --git a/packages/api/src/services/save_page.ts b/packages/api/src/services/save_page.ts index daa718729..9764690b2 100644 --- a/packages/api/src/services/save_page.ts +++ b/packages/api/src/services/save_page.ts @@ -92,6 +92,7 @@ export const savePage = async ( saveTime: input.savedAt ? new Date(input.savedAt) : undefined, publishedAt: input.publishedAt ? new Date(input.publishedAt) : undefined, state: input.state || undefined, + rssFeedUrl: input.rssFeedUrl, }) const isImported = input.source === 'csv-importer' @@ -122,17 +123,14 @@ export const savePage = async ( // check if the page already exists const existingLibraryItem = await authTrx((t) => - t.getRepository(LibraryItem).findOne({ - where: { user: { id: user.id }, originalUrl: itemToSave.originalUrl }, - relations: ['subscription'], + t.getRepository(LibraryItem).findOneBy({ + user: { id: user.id }, + originalUrl: itemToSave.originalUrl, }) ) if (existingLibraryItem) { // we don't want to update an rss feed page if rss-feeder is tring to re-save it - if ( - existingLibraryItem.subscription && - existingLibraryItem.subscription.url === input.rssFeedUrl - ) { + if (existingLibraryItem.subscription === input.rssFeedUrl) { return { clientRequestId, url: `${homePageURL()}/${user.profile.username}/${slug}`, @@ -206,6 +204,7 @@ export const parsedContentToLibraryItem = ({ saveTime, publishedAt, state, + rssFeedUrl, }: { url: string userId: string @@ -223,6 +222,7 @@ export const parsedContentToLibraryItem = ({ saveTime?: Date publishedAt?: Date | null state?: ArticleSavingRequestStatus | null + rssFeedUrl?: string | null }): DeepPartial & { originalUrl: string } => { return { id: itemId || undefined, @@ -260,5 +260,6 @@ export const parsedContentToLibraryItem = ({ siteIcon: parsedContent?.siteIcon, wordCount: wordsCount(parsedContent?.textContent || ''), contentReader: contentReaderForLibraryItem(itemType, uploadFileId), + subscription: rssFeedUrl, } } diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts index 4b90d217e..3ff472853 100644 --- a/packages/api/src/utils/createTask.ts +++ b/packages/api/src/utils/createTask.ts @@ -442,15 +442,15 @@ export const enqueueTextToSpeech = async ({ export const enqueueRecommendation = async ( userId: string, - pageId: string, - recommendation: Recommendation, + itemId: string, + recommendation: Partial, authToken: string, highlightIds?: string[] ): Promise => { const { GOOGLE_CLOUD_PROJECT } = process.env const payload = { userId, - pageId, + itemId, recommendation, highlightIds, } diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 5a4cf5d59..ac0453084 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -211,7 +211,13 @@ const recommandationDataToRecommendation = ( recommendation: RecommendationData ): Recommendation => ({ ...recommendation, - name: recommendation.recommender.name, + user: { + userId: recommendation.recommender.id, + username: recommendation.recommender.profile.username, + profileImageURL: recommendation.recommender.profile.pictureUrl, + name: recommendation.recommender.name, + }, + name: recommendation.group.name, recommendedAt: recommendation.createdAt, }) @@ -236,7 +242,6 @@ export const libraryItemToArticle = (item: LibraryItem): Article => ({ recommendations: item.recommendations?.map( recommandationDataToRecommendation ), - subscription: item.subscription?.name, image: item.thumbnail, contentReader: item.contentReader as unknown as ContentReader, readingProgressAnchorIndex: item.readingProgressHighestReadAnchor, @@ -256,7 +261,6 @@ export const libraryItemToSearchItem = (item: LibraryItem): SearchItem => ({ readingProgressPercent: item.readingProgressBottomPercent, contentReader: item.contentReader as unknown as ContentReader, readingProgressAnchorIndex: item.readingProgressHighestReadAnchor, - subscription: item.subscription?.name, recommendations: item.recommendations?.map( recommandationDataToRecommendation ), diff --git a/packages/db/elastic_migrations/migrate_from_elastic.py b/packages/db/elastic_migrations/migrate_from_elastic.py new file mode 100644 index 000000000..5334e0a16 --- /dev/null +++ b/packages/db/elastic_migrations/migrate_from_elastic.py @@ -0,0 +1,129 @@ +#!/usr/bin/python +import hashlib +import os +import uuid +from collections import deque + +import pandas as pd +from elasticsearch import Elasticsearch +from elasticsearch.helpers import scan as escan + +PG_HOST = os.getenv('PG_HOST', 'localhost') +PG_PORT = os.getenv('PG_PORT', 5432) +PG_USER = os.getenv('PG_USER', 'app_user') +PG_PASSWORD = os.getenv('PG_PASSWORD', 'app_pass') +PG_DB = os.getenv('PG_DB', 'omnivore') +ES_URL = os.getenv('ES_URL', 'http://localhost:9200') +ES_USERNAME = os.getenv('ES_USERNAME', 'elastic') +ES_PASSWORD = os.getenv('ES_PASSWORD', 'password') +ES_SCAN_SIZE = os.getenv('ES_SCAN_SIZE', 100) +ES_INDEX = os.getenv('ES_INDEX', 'pages_alias') + +CUT_OFF_DATE = os.getenv('CUT_OFF_DATE', '2000-01-01') + + +def convert_string_to_uuid(val: str): + hex_string = hashlib.md5(val.encode('UTF-8')).hexdigest() + return uuid.UUID(hex=hex_string) + + +# def assertData(conn, client): +# # get all users from postgres +# try: +# success = 0 +# failure = 0 +# cursor = conn.cursor(cursor_factory=RealDictCursor) +# cursor.execute('''SELECT id FROM omnivore.user''') +# result = cursor.fetchall() +# for row in result: +# userId = row['id'] +# cursor.execute( +# f'SELECT COUNT(*) FROM omnivore.links WHERE user_id = \'{userId}\'''') +# countInPostgres = cursor.fetchone()['count'] +# countInElastic = client.count( +# index='pages_alias', body={'query': {'term': {'userId': userId}}})['count'] + +# if countInPostgres == countInElastic: +# success += 1 +# print(f'User {userId} OK') +# else: +# failure += 1 +# print( +# f'User {userId} ERROR: postgres: {countInPostgres}, elastic: {countInElastic}') +# cursor.close() +# print(f'Asserted data, success: {success}, failure: {failure}') +# except Exception as err: +# print('Assert data ERROR:', err) +# exit(1) + + +def update_postgres_data(conn, query, table): + try: + print('Executing query: {}'.format(query)) + # update data in postgres + cursor = conn.cursor() + cursor.execute(query) + count = cursor.rowcount + conn.commit() + cursor.close() + print(f'Updated {table} in postgres, rows: ', count) + except Exception as err: + print('Update postgres data ERROR:', err) + + +def get_data_from_es(): + # elastic client + client = Elasticsearch(ES_URL, http_auth=( + ES_USERNAME, ES_PASSWORD), retry_on_timeout=True) + try: + print('Elasticsearch client connected', client.info()) + except Exception as err: + print('Elasticsearch client ERROR:', err) + exit(1) + + query = { + "query": { + "bool": { + "must": [ + { + "range": { + "updatedAt": { + "gte": CUT_OFF_DATE + } + } + } + ] + } + }, + "sort": [ + { + "updatedAt": { + "order": "desc" + } + } + ] + } + # Scan API for larger library + response = escan(client=client, index=ES_INDEX, query=query, + preserve_order=True, size=ES_SCAN_SIZE, + request_timeout=30) + + # Initialize a double ended queue + output = deque() + # Extend deque with iterator + output.extend(response) + # Convert deque to DataFrame + df = pd.json_normalize(output) + df = df[[x for x in df.columns if "_source." in x or x == '_id']] + + client.close() + return df + + +print('Starting migration') + +# get data from elastic +df = get_data_from_es() +print(df.head()) + +print('Migration complete') diff --git a/packages/db/migrations/0118.do.library_item.sql b/packages/db/migrations/0118.do.library_item.sql index 7a9cdef06..8a33fd990 100755 --- a/packages/db/migrations/0118.do.library_item.sql +++ b/packages/db/migrations/0118.do.library_item.sql @@ -54,7 +54,7 @@ CREATE TABLE omnivore.library_item ( text_content_hash text, gcs_archive_id text, directionality directionality_type NOT NULL DEFAULT 'LTR', - subscription_id uuid REFERENCES omnivore.subscriptions ON DELETE CASCADE, + subscription text, label_names text[] NOT NULL DEFAULT array[]::text[], -- array of label names of the item highlight_labels text[] NOT NULL DEFAULT array[]::text[], -- array of label names of the item's highlights highlight_annotations text[] NOT NULL DEFAULT array[]::text[], -- array of highlight annotations of the item diff --git a/packages/db/migrations/0119.undo.add_defaults_to_filters.sql b/packages/db/migrations/0119.undo.add_defaults_to_filters.sql index 3e30d524a..9cb6448f9 100644 --- a/packages/db/migrations/0119.undo.add_defaults_to_filters.sql +++ b/packages/db/migrations/0119.undo.add_defaults_to_filters.sql @@ -4,7 +4,7 @@ BEGIN; ALTER TABLE omnivore.filters - DROP COLUMN default, + DROP COLUMN default_filter, DROP COLUMN visible; COMMIT; diff --git a/packages/db/migrations/0122.do.update_highlight.sql b/packages/db/migrations/0122.do.update_highlight.sql index 2e2a9829a..07bad74f5 100755 --- a/packages/db/migrations/0122.do.update_highlight.sql +++ b/packages/db/migrations/0122.do.update_highlight.sql @@ -6,8 +6,7 @@ BEGIN; CREATE TYPE highlight_type AS ENUM ( 'HIGHLIGHT', - 'REDACTION', - 'NOTE' + 'REDACTION' ); ALTER TABLE omnivore.highlight @@ -19,15 +18,10 @@ ALTER TABLE omnivore.highlight ADD COLUMN html text, ALTER COLUMN quote DROP NOT NULL, ALTER COLUMN patch DROP NOT NULL, + DROP COLUMN deleted, DROP COLUMN article_id, DROP COLUMN elastic_page_id; -ALTER POLICY read_highlight on omnivore.highlight - USING (user_id = omnivore.get_current_user_id()); - -ALTER POLICY create_highlight on omnivore.highlight - WITH CHECK (user_id = omnivore.get_current_user_id()); - CREATE POLICY delete_highlight on omnivore.highlight FOR DELETE TO omnivore_user USING (user_id = omnivore.get_current_user_id()); diff --git a/packages/db/migrations/0122.undo.update_highlight.sql b/packages/db/migrations/0122.undo.update_highlight.sql index 5bc050127..7a5c10dfd 100755 --- a/packages/db/migrations/0122.undo.update_highlight.sql +++ b/packages/db/migrations/0122.undo.update_highlight.sql @@ -7,12 +7,11 @@ BEGIN; DROP TRIGGER IF EXISTS library_item_highlight_annotations_update ON omnivore.highlight; DROP FUNCTION IF EXISTS update_library_item_highlight_annotations(); -ALTER POLICY read_highlight on omnivore.highlight USING (true); -ALTER POLICY create_highlight on omnivore.highlight WITH CHECK (true); DROP POLICY delete_highlight on omnivore.highlight; REVOKE DELETE ON omnivore.highlight FROM omnivore_user; ALTER TABLE omnivore.highlight + ADD COLUMN deleted boolean DEFAULT false, ADD COLUMN article_id uuid, ADD COLUMN elastic_page_id uuid, DROP COLUMN library_item_id, diff --git a/packages/db/migrations/0123.do.recommendation.sql b/packages/db/migrations/0123.do.recommendation.sql index 08efba968..e201acd86 100755 --- a/packages/db/migrations/0123.do.recommendation.sql +++ b/packages/db/migrations/0123.do.recommendation.sql @@ -8,8 +8,10 @@ CREATE TABLE omnivore.recommendation ( id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(), library_item_id uuid NOT NULL REFERENCES omnivore.library_item ON DELETE CASCADE, recommender_id uuid NOT NULL REFERENCES omnivore.user ON DELETE CASCADE, + group_id uuid NOT NULL REFERENCES omnivore.group ON DELETE CASCADE, note text, - created_at timestamptz NOT NULL DEFAULT current_timestamp + created_at timestamptz NOT NULL DEFAULT current_timestamp, + UNIQUE (library_item_id, recommender_id, group_id) ); GRANT SELECT, INSERT ON omnivore.recommendation TO omnivore_user;