From 2e26a04cd09736131a1caaf88fa74dd790dd83fa Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 20 Jun 2024 15:47:52 +0800 Subject: [PATCH 01/32] remove redundant context type --- .../api/src/resolvers/function_resolvers.ts | 86 +++------- packages/api/src/resolvers/report/index.ts | 4 +- packages/api/src/resolvers/types.ts | 5 - packages/api/src/resolvers/user/index.ts | 16 +- .../src/resolvers/user_feed_article/index.ts | 150 ------------------ .../api/src/resolvers/user_friends/index.ts | 101 ------------ packages/api/src/utils/gql-utils.ts | 6 +- 7 files changed, 39 insertions(+), 329 deletions(-) delete mode 100644 packages/api/src/resolvers/user_feed_article/index.ts delete mode 100644 packages/api/src/resolvers/user_friends/index.ts diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index febe9deb0..a406ae707 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -157,7 +157,7 @@ import { } from './recent_emails' import { recentSearchesResolver } from './recent_searches' import { subscriptionResolver } from './subscriptions' -import { WithDataSourcesContext } from './types' +import { ResolverContext } from './types' import { updateEmailResolver } from './user' /* eslint-disable @typescript-eslint/naming-convention */ @@ -180,7 +180,7 @@ const readingProgressHandlers = { async readingProgressPercent( article: LibraryItem, _: unknown, - ctx: WithDataSourcesContext + ctx: ResolverContext ) { if (ctx.claims?.uid) { const readingProgress = @@ -200,7 +200,7 @@ const readingProgressHandlers = { async readingProgressAnchorIndex( article: LibraryItem, _: unknown, - ctx: WithDataSourcesContext + ctx: ResolverContext ) { if (ctx.claims?.uid) { const readingProgress = @@ -220,7 +220,7 @@ const readingProgressHandlers = { async readingProgressTopPercent( article: LibraryItem, _: unknown, - ctx: WithDataSourcesContext + ctx: ResolverContext ) { if (ctx.claims?.uid) { const readingProgress = @@ -364,11 +364,7 @@ export const functionResolvers = { } return undefined }, - async features( - _: User, - __: Record, - ctx: WithDataSourcesContext - ) { + async features(_: User, __: Record, ctx: ResolverContext) { if (!ctx.claims?.uid) { return undefined } @@ -378,7 +374,7 @@ export const functionResolvers = { async featureList( _: User, __: Record, - ctx: WithDataSourcesContext + ctx: ResolverContext ) { if (!ctx.claims?.uid) { return undefined @@ -398,7 +394,7 @@ export const functionResolvers = { sharedNotesCount: () => 0, }, Article: { - async url(article: LibraryItem, _: unknown, ctx: WithDataSourcesContext) { + async url(article: LibraryItem, _: unknown, ctx: ResolverContext) { if ( (article.itemType == PageType.File || article.itemType == PageType.Book) && @@ -439,20 +435,12 @@ export const functionResolvers = { ? wordsCount(article.readableContent) : undefined }, - async labels( - article: LibraryItem, - _: unknown, - ctx: WithDataSourcesContext - ) { + async labels(article: LibraryItem, _: unknown, ctx: ResolverContext) { if (article.labels) return article.labels return ctx.dataLoaders.labels.load(article.id) }, - async highlights( - article: LibraryItem, - _: unknown, - ctx: WithDataSourcesContext - ) { + async highlights(article: LibraryItem, _: unknown, ctx: ResolverContext) { if (article.highlights) return article.highlights return ctx.dataLoaders.highlights.load(article.id) @@ -468,35 +456,27 @@ export const functionResolvers = { reactions: () => [], replies: () => [], type: (highlight: Highlight) => highlight.highlightType, - async user(highlight: Highlight, __: unknown, ctx: WithDataSourcesContext) { + async user(highlight: Highlight, __: unknown, ctx: ResolverContext) { return ctx.dataLoaders.users.load(highlight.userId) }, - createdByMe( - highlight: Highlight, - __: unknown, - ctx: WithDataSourcesContext - ) { - return highlight.userId === ctx.uid + createdByMe(highlight: Highlight, __: unknown, ctx: ResolverContext) { + return highlight.userId === ctx.claims?.uid }, - libraryItem(highlight: Highlight, _: unknown, ctx: WithDataSourcesContext) { + libraryItem(highlight: Highlight, _: unknown, ctx: ResolverContext) { if (highlight.libraryItem) { return highlight.libraryItem } return ctx.dataLoaders.libraryItems.load(highlight.libraryItemId) }, - labels: async ( - highlight: Highlight, - _: unknown, - ctx: WithDataSourcesContext - ) => { + labels: async (highlight: Highlight, _: unknown, ctx: ResolverContext) => { return ( highlight.labels || ctx.dataLoaders.highlightLabels.load(highlight.id) ) }, }, SearchItem: { - async url(item: LibraryItem, _: unknown, ctx: WithDataSourcesContext) { + async url(item: LibraryItem, _: unknown, ctx: ResolverContext) { if ( (item.itemType == PageType.File || item.itemType == PageType.Book) && ctx.claims && @@ -528,47 +508,33 @@ export const functionResolvers = { return item.siteIcon }, - async labels(item: LibraryItem, _: unknown, ctx: WithDataSourcesContext) { + async labels(item: LibraryItem, _: unknown, ctx: ResolverContext) { if (item.labels) return item.labels return ctx.dataLoaders.labels.load(item.id) }, - async recommendations( - item: LibraryItem, - _: unknown, - ctx: WithDataSourcesContext - ) { + async recommendations(item: LibraryItem, _: unknown, ctx: ResolverContext) { if (item.recommendations) return item.recommendations return ctx.dataLoaders.recommendations.load(item.id) }, - async aiSummary( - item: LibraryItem, - _: unknown, - ctx: WithDataSourcesContext - ) { + async aiSummary(item: LibraryItem, _: unknown, ctx: ResolverContext) { + if (!ctx.claims) return undefined + return ( await getAISummary({ - userId: ctx.uid, + userId: ctx.claims.uid, libraryItemId: item.id, idx: 'latest', }) )?.summary }, - async highlights( - item: LibraryItem, - _: unknown, - ctx: WithDataSourcesContext - ) { + async highlights(item: LibraryItem, _: unknown, ctx: ResolverContext) { if (item.highlights) return item.highlights return ctx.dataLoaders.highlights.load(item.id) }, - async content( - item: PartialLibraryItem, - _: unknown, - ctx: WithDataSourcesContext - ) { + async content(item: PartialLibraryItem, _: unknown, ctx: ResolverContext) { // convert html to the requested format if requested if ( item.format && @@ -658,7 +624,7 @@ export const functionResolvers = { }> }, _: unknown, - ctx: WithDataSourcesContext + ctx: ResolverContext ) { const items = section.items @@ -745,7 +711,7 @@ export const functionResolvers = { { subscription?: string; siteName: string; siteIcon?: string } >, _: unknown, - ctx: WithDataSourcesContext + ctx: ResolverContext ): Promise { if (item.source) { return item.source @@ -785,7 +751,7 @@ export const functionResolvers = { ArticleSavingRequest: { status: (item: LibraryItem) => item.state, url: (item: LibraryItem) => item.originalUrl, - async user(_item: LibraryItem, __: unknown, ctx: WithDataSourcesContext) { + async user(_item: LibraryItem, __: unknown, ctx: ResolverContext) { if (ctx.claims?.uid) { return ctx.dataLoaders.users.load(ctx.claims.uid) } diff --git a/packages/api/src/resolvers/report/index.ts b/packages/api/src/resolvers/report/index.ts index 6ccbf76a5..e4b5eeb90 100644 --- a/packages/api/src/resolvers/report/index.ts +++ b/packages/api/src/resolvers/report/index.ts @@ -11,7 +11,7 @@ import { saveContentDisplayReport, } from '../../services/reports' import { analytics } from '../../utils/analytics' -import { WithDataSourcesContext } from '../types' +import { ResolverContext } from '../types' const SUCCESS_MESSAGE = `Your report has been submitted. Thank you.` const FAILURE_MESSAGE = @@ -36,7 +36,7 @@ const isContentDisplayReport = (types: ReportType[]): boolean => { export const reportItemResolver: ResolverFn< ReportItemResult, unknown, - WithDataSourcesContext, + ResolverContext, MutationReportItemArgs > = async (_obj, args, ctx) => { const { sharedBy, reportTypes } = args.input diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts index 16857d85d..47fa91954 100644 --- a/packages/api/src/resolvers/types.ts +++ b/packages/api/src/resolvers/types.ts @@ -14,7 +14,6 @@ import { Recommendation } from '../entity/recommendation' import { Subscription } from '../entity/subscription' import { UploadFile } from '../entity/upload_file' import { User } from '../entity/user' -import { HomeItem } from '../generated/graphql' import { PubsubClient } from '../pubsub' export interface Claims { @@ -65,7 +64,3 @@ export interface RequestContext { } export type ResolverContext = ApolloContext - -export type WithDataSourcesContext = { - uid: string -} & ResolverContext diff --git a/packages/api/src/resolvers/user/index.ts b/packages/api/src/resolvers/user/index.ts index 6572c1f10..9c496df8c 100644 --- a/packages/api/src/resolvers/user/index.ts +++ b/packages/api/src/resolvers/user/index.ts @@ -45,7 +45,7 @@ import { softDeleteUser } from '../../services/user' import { Merge } from '../../util' import { authorized } from '../../utils/gql-utils' import { validateUsername } from '../../utils/usernamePolicy' -import { WithDataSourcesContext } from '../types' +import { ResolverContext } from '../types' export const updateUserResolver = authorized< Merge, @@ -145,7 +145,7 @@ export const updateUserProfileResolver = authorized< export const googleLoginResolver: ResolverFn< Merge, unknown, - WithDataSourcesContext, + ResolverContext, MutationGoogleLoginArgs > = async (_obj, { input }, { setAuth }) => { const { email, secret } = input @@ -172,7 +172,7 @@ export const googleLoginResolver: ResolverFn< export const validateUsernameResolver: ResolverFn< boolean, Record, - WithDataSourcesContext, + ResolverContext, QueryValidateUsernameArgs > = async (_obj, { username }) => { const lowerCasedUsername = username.toLowerCase() @@ -191,7 +191,7 @@ export const validateUsernameResolver: ResolverFn< export const googleSignupResolver: ResolverFn< Merge, Record, - WithDataSourcesContext, + ResolverContext, MutationGoogleSignupArgs > = async (_obj, { input }, { setAuth, log }) => { const { email, username, name, bio, sourceUserId, pictureUrl, secret } = input @@ -231,7 +231,7 @@ export const googleSignupResolver: ResolverFn< export const logOutResolver: ResolverFn< LogOutResult, unknown, - WithDataSourcesContext, + ResolverContext, unknown > = (_, __, { clearAuth, log }) => { try { @@ -246,7 +246,7 @@ export const logOutResolver: ResolverFn< export const getMeUserResolver: ResolverFn< UserEntity | undefined, unknown, - WithDataSourcesContext, + ResolverContext, unknown > = async (_obj, __, { claims }) => { try { @@ -268,9 +268,9 @@ export const getMeUserResolver: ResolverFn< export const getUserResolver: ResolverFn< Merge, unknown, - WithDataSourcesContext, + ResolverContext, QueryUserArgs -> = async (_obj, { userId: id, username }, { uid }) => { +> = async (_obj, { userId: id, username }) => { if (!(id || username)) { return { errorCodes: [UserErrorCode.BadRequest] } } diff --git a/packages/api/src/resolvers/user_feed_article/index.ts b/packages/api/src/resolvers/user_feed_article/index.ts deleted file mode 100644 index 928078f9d..000000000 --- a/packages/api/src/resolvers/user_feed_article/index.ts +++ /dev/null @@ -1,150 +0,0 @@ -/* eslint-disable @typescript-eslint/require-await */ -/* eslint-disable @typescript-eslint/no-unused-vars */ -import { FeedArticle, PageInfo } from '../../generated/graphql' - -export type PartialFeedArticle = Omit< - FeedArticle, - 'sharedBy' | 'article' | 'reactions' -> - -type PaginatedFeedArticlesSuccessPartial = { - edges: { cursor: string; node: PartialFeedArticle }[] - pageInfo: PageInfo -} - -// export const getSharedArticleResolver: ResolverFn< -// SharedArticleSuccessPartial | SharedArticleError, -// Record, -// WithDataSourcesContext, -// QuerySharedArticleArgs -// > = async (_obj, { username, slug, selectedHighlightId }, { kx, models }) => { -// try { -// const user = await models.user.getWhere({ username }) -// if (!user) { -// return { -// errorCodes: [SharedArticleErrorCode.NotFound], -// } -// } - -// const article = await models.userArticle.getBySlug(username, slug) -// if (!article || !article.sharedAt) { -// return { -// errorCodes: [SharedArticleErrorCode.NotFound], -// } -// } - -// if (selectedHighlightId) { -// const highlightResult = await models.highlight.getWhereIn('shortId', [ -// selectedHighlightId, -// ]) -// if (!highlightResult || !highlightResult[0].sharedAt) { -// return { -// errorCodes: [SharedArticleErrorCode.NotFound], -// } -// } -// } - -// const shareInfo = await getShareInfoForArticle( -// kx, -// user.id, -// article.id, -// models -// ) - -// return { article: { ...article, userId: user.id, shareInfo: shareInfo } } -// } catch (error) { -// return { errorCodes: [SharedArticleErrorCode.NotFound] } -// } -// } - -// export const getUserFeedArticlesResolver: ResolverFn< -// PaginatedFeedArticlesSuccessPartial, -// unknown, -// WithDataSourcesContext, -// QueryFeedArticlesArgs -// > = async ( -// _obj, -// { after: _startCursor, first: _first, sharedByUser }, -// { models, claims, authTrx } -// ) => { -// if (!(sharedByUser || claims?.uid)) { -// return { -// edges: [], -// pageInfo: { -// startCursor: '', -// endCursor: '', -// hasNextPage: false, -// hasPreviousPage: false, -// }, -// } -// } - -// const first = _first || 0 -// const startCursor = _startCursor || '' - -// const feedArticles = -// (await authTrx((tx) => -// models.userArticle.getUserFeedArticlesPaginatedWithHighlights( -// { cursor: startCursor, first: first + 1, sharedByUser }, // fetch one more item to get next cursor -// claims?.uid || '', -// tx -// ) -// )) || [] - -// const endCursor = feedArticles[feedArticles.length - 1]?.sharedAt -// .getTime() -// ?.toString() -// const hasNextPage = feedArticles.length > first - -// if (hasNextPage) { -// // remove an extra if exists -// feedArticles.pop() -// } - -// const edges = feedArticles.map((fa) => { -// return { -// node: fa, -// cursor: fa.sharedAt.getTime()?.toString(), -// } -// }) - -// return { -// edges, -// pageInfo: { -// hasPreviousPage: false, -// startCursor: '', -// hasNextPage, -// endCursor, -// }, -// } -// } - -// export const updateSharedCommentResolver = authorized< -// UpdateSharedCommentSuccess, -// UpdateSharedCommentError, -// MutationUpdateSharedCommentArgs -// >( -// async ( -// _, -// { input: { articleID, sharedComment } }, -// { models, authTrx, claims: { uid } } -// ) => { -// const ua = await authTrx((tx) => -// models.userArticle.getByParameters(uid, { articleId: articleID }, tx) -// ) -// if (!ua) { -// return { errorCodes: [UpdateSharedCommentErrorCode.NotFound] } -// } - -// await authTrx((tx) => -// models.userArticle.updateByArticleId( -// uid, -// articleID, -// { sharedComment }, -// tx -// ) -// ) - -// return { articleID, sharedComment } -// } -// ) diff --git a/packages/api/src/resolvers/user_friends/index.ts b/packages/api/src/resolvers/user_friends/index.ts deleted file mode 100644 index 823184066..000000000 --- a/packages/api/src/resolvers/user_friends/index.ts +++ /dev/null @@ -1,101 +0,0 @@ -// export const setFollowResolver = authorized< -// SetFollowSuccess, -// SetFollowError, -// MutationSetFollowArgs -// >( -// async ( -// _, -// { input: { userId: friendUserId, follow } }, -// { models, authTrx, claims: { uid } } -// ) => { -// const user = await models.user.getUserDetails(uid, friendUserId) -// if (!user) return { errorCodes: [SetFollowErrorCode.NotFound] } - -// const userFriendRecord = await authTrx((tx) => -// models.userFriends.getByUserFriendId(uid, friendUserId, tx) -// ) - -// if (follow) { -// if (!userFriendRecord) { -// await authTrx((tx) => -// models.userFriends.create({ friendUserId, userId: uid }, tx) -// ) -// } -// } else if (userFriendRecord) { -// await authTrx((tx) => models.userFriends.delete(userFriendRecord.id, tx)) -// } - -// const updatedUser = await models.user.getUserDetails(uid, friendUserId) -// if (!updatedUser) return { errorCodes: [SetFollowErrorCode.NotFound] } - -// return { -// updatedUser: { -// ...userDataToUser(updatedUser), -// isFriend: updatedUser.viewerIsFollowing, -// }, -// } -// } -// ) - -// const getUserList = async ( -// uid: string, -// users: UserData[], -// models: DataModels, -// authTrx: ( -// cb: (tx: Knex.Transaction) => TResult, -// userRole?: string -// ) => Promise -// ): Promise => { -// const usersIds = users.map(({ id }) => id) -// const friends = await authTrx((tx) => -// models.userFriends.getByFriendIds(uid, usersIds, tx) -// ) - -// const friendsIds = friends.map(({ friendUserId }) => friendUserId) -// users = users.map((f) => ({ -// ...f, -// isFriend: friendsIds.includes(f.id), -// viewerIsFollowing: friendsIds.includes(f.id), -// })) - -// return users.map((u) => userDataToUser(u)) -// } - -// export const getFollowersResolver: ResolverFn< -// GetFollowersResult, -// unknown, -// WithDataSourcesContext, -// QueryGetFollowersArgs -// > = async (_parent, { userId }, { models, claims, authTrx }) => { -// const followers = userId -// ? await authTrx((tx) => models.user.getUserFollowersList(userId, tx)) -// : [] -// if (!claims?.uid) return { followers: usersWithNoFriends(followers) } -// return { -// followers: await getUserList(claims?.uid, followers, models, authTrx), -// } -// } - -// export const getFollowingResolver: ResolverFn< -// GetFollowingResult, -// unknown, -// WithDataSourcesContext, -// QueryGetFollowingArgs -// > = async (_parent, { userId }, { models, claims, authTrx }) => { -// const following = userId -// ? await authTrx((tx) => models.user.getUserFollowingList(userId, tx)) -// : [] -// if (!claims?.uid) return { following: usersWithNoFriends(following) } -// return { -// following: await getUserList(claims?.uid, following, models, authTrx), -// } -// } - -// const usersWithNoFriends = (users: UserData[]): User[] => { -// return users.map((f) => -// userDataToUser({ -// ...f, -// isFriend: false, -// } as UserData) -// ) -// } diff --git a/packages/api/src/utils/gql-utils.ts b/packages/api/src/utils/gql-utils.ts index ef5986f28..62bbf2d09 100644 --- a/packages/api/src/utils/gql-utils.ts +++ b/packages/api/src/utils/gql-utils.ts @@ -1,5 +1,5 @@ import { ResolverFn } from '../generated/graphql' -import { Claims, WithDataSourcesContext } from '../resolvers/types' +import { Claims, ResolverContext } from '../resolvers/types' export function authorized< TSuccess, @@ -12,10 +12,10 @@ export function authorized< resolver: ResolverFn< TSuccess | TError, TParent, - WithDataSourcesContext & { claims: Claims }, + ResolverContext & { claims: Claims; uid: string }, TArgs > -): ResolverFn { +): ResolverFn { return (parent, args, ctx, info) => { const { claims } = ctx if (claims?.uid) { From 40182421f740be4d6542c3b53762949cae8a80db Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 20 Jun 2024 16:31:05 +0800 Subject: [PATCH 02/32] show archived highlights --- .../api/src/resolvers/function_resolvers.ts | 11 ++++++-- packages/api/src/services/highlights.ts | 1 - packages/api/src/services/library_item.ts | 27 ++++--------------- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts index a406ae707..67624303a 100644 --- a/packages/api/src/resolvers/function_resolvers.ts +++ b/packages/api/src/resolvers/function_resolvers.ts @@ -6,7 +6,7 @@ import { createHmac } from 'crypto' import { isError } from 'lodash' import { Highlight } from '../entity/highlight' -import { LibraryItem } from '../entity/library_item' +import { LibraryItem, LibraryItemState } from '../entity/library_item' import { EXISTING_NEWSLETTER_FOLDER, NewsletterEmail, @@ -634,7 +634,14 @@ export const functionResolvers = { const libraryItems = ( await ctx.dataLoaders.libraryItems.loadMany(libraryItemIds) ).filter( - (libraryItem) => !!libraryItem && !isError(libraryItem) + (libraryItem) => + !!libraryItem && + !isError(libraryItem) && + [ + LibraryItemState.Succeeded, + LibraryItemState.ContentNotFetched, + ].includes(libraryItem.state) && + !libraryItem.seenAt ) as Array const publicItemIds = section.items diff --git a/packages/api/src/services/highlights.ts b/packages/api/src/services/highlights.ts index 327038aca..533d29eb2 100644 --- a/packages/api/src/services/highlights.ts +++ b/packages/api/src/services/highlights.ts @@ -26,7 +26,6 @@ export const batchGetHighlightsFromLibraryItemIds = async ( const highlights = await authTrx(async (tx) => tx.getRepository(Highlight).find({ where: { libraryItem: { id: In(libraryItemIds as string[]) } }, - relations: ['user'], }) ) diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts index 4ed0136a6..4eee6af0c 100644 --- a/packages/api/src/services/library_item.ts +++ b/packages/api/src/services/library_item.ts @@ -6,7 +6,6 @@ import { EntityManager, FindOptionsWhere, In, - IsNull, ObjectLiteral, } from 'typeorm' import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity' @@ -135,31 +134,15 @@ export enum SortBy { const readingProgressDataSource = new ReadingProgressDataSource() export const batchGetLibraryItems = async (ids: readonly string[]) => { - const selectColumns: Array = [ - 'id', - 'title', - 'author', - 'thumbnail', - 'wordCount', - 'savedAt', - 'originalUrl', - 'directionality', - 'description', - 'subscription', - 'siteName', - 'siteIcon', - 'archivedAt', - 'deletedAt', - 'slug', - 'previewContent', - ] + // select all columns except content + const select = getColumns(libraryItemRepository).filter( + (select) => ['originalContent', 'readableContent'].indexOf(select) === -1 + ) const items = await authTrx(async (tx) => tx.getRepository(LibraryItem).find({ - select: selectColumns, + select, where: { id: In(ids as string[]), - state: LibraryItemState.Succeeded, - seenAt: IsNull(), }, }) ) From 8c84fc58b8786b99b93955f763979c17bae4a017 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Thu, 20 Jun 2024 17:36:57 +0800 Subject: [PATCH 03/32] add more subscription features to the score api payload --- packages/api/src/jobs/update_home.ts | 20 +++++++++++++++++++- packages/api/src/services/score.ts | 10 ++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts index 661b235d0..b565ed7a2 100644 --- a/packages/api/src/jobs/update_home.ts +++ b/packages/api/src/jobs/update_home.ts @@ -1,7 +1,7 @@ import client from 'prom-client' import { LibraryItem } from '../entity/library_item' import { PublicItem } from '../entity/public_item' -import { Subscription } from '../entity/subscription' +import { Subscription, SubscriptionType } from '../entity/subscription' import { User } from '../entity/user' import { registerMetric } from '../prometheus' import { redisDataSource } from '../redis_data_source' @@ -41,6 +41,9 @@ interface Candidate { subscription?: { name: string type: string + autoAddToLibrary?: boolean | null + createdAt: Date + fetchContent?: boolean | null } } @@ -102,6 +105,7 @@ const publicItemToCandidate = (item: PublicItem): Candidate => ({ subscription: { name: item.source.name, type: item.source.type, + createdAt: item.source.createdAt, }, score: 0, }) @@ -222,6 +226,20 @@ const rankCandidates = async ( word_count: item.wordCount, published_at: item.publishedAt, subscription: item.subscription?.name, + inbox_folder: item.folder === 'inbox', + is_feed: item.subscription?.type === SubscriptionType.Rss, + is_newsletter: item.subscription?.type === SubscriptionType.Newsletter, + is_subscription: !!item.subscription, + item_word_count: item.wordCount, + subscription_count: 0, + subscription_auto_add_to_library: item.subscription?.autoAddToLibrary, + subscription_fetch_content: item.subscription?.fetchContent, + days_since_subscribed: item.subscription + ? Math.floor( + (Date.now() - item.subscription.createdAt.getTime()) / + (1000 * 60 * 60 * 24) + ) + : undefined, } as Feature return acc }, {} as Record), diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts index c8c8c1660..25c78f5a0 100644 --- a/packages/api/src/services/score.ts +++ b/packages/api/src/services/score.ts @@ -6,6 +6,12 @@ export interface Feature { has_thumbnail: boolean has_site_icon: boolean saved_at: Date + item_word_count: number + is_subscription: boolean + inbox_folder: boolean + is_newsletter: boolean + is_feed: boolean + site?: string language?: string author?: string @@ -15,6 +21,10 @@ export interface Feature { folder?: string published_at?: Date subscription?: string + subscription_auto_add_to_library?: boolean + subscription_fetch_content?: boolean + days_since_subscribed?: number + subscription_count?: number } export interface ScoreApiRequestBody { From 17e59787924ae8f11fdd2810534fc49927a1510e Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 20 Jun 2024 15:49:12 +0800 Subject: [PATCH 04/32] Update digest-score to run new model --- ml/digest-score/Dockerfile | 13 + ml/digest-score/app.py | 286 ++++-------- ml/digest-score/features.py | 31 ++ ml/digest-score/features/__init__.py | 0 ml/digest-score/features/extract.py | 109 +++++ ml/digest-score/features/user_history.py | 235 ++++++++++ ml/digest-score/requirements.txt | 2 + ml/digest-score/train.py | 415 +++++++----------- .../resolvers/article_saving_request.test.ts | 12 +- 9 files changed, 638 insertions(+), 465 deletions(-) create mode 100644 ml/digest-score/Dockerfile create mode 100644 ml/digest-score/features.py create mode 100644 ml/digest-score/features/__init__.py create mode 100644 ml/digest-score/features/extract.py create mode 100644 ml/digest-score/features/user_history.py diff --git a/ml/digest-score/Dockerfile b/ml/digest-score/Dockerfile new file mode 100644 index 000000000..0afa6baec --- /dev/null +++ b/ml/digest-score/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.8-slim + +WORKDIR /app + +ENV GRPC_PYTHON_BUILD_SYSTEM_OPENSSL "1" +ENV GRPC_PYTHON_BUILD_SYSTEM_ZLIB "1" + +COPY . /app + +RUN pip install --no-cache-dir -r requirements.txt + +EXPOSE 5000 +CMD ["python", "serve.py"] diff --git a/ml/digest-score/app.py b/ml/digest-score/app.py index 0d1765525..501a417be 100644 --- a/ml/digest-score/app.py +++ b/ml/digest-score/app.py @@ -1,8 +1,5 @@ -import psycopg2 - import logging from flask import Flask, request, jsonify -from pydantic import BaseModel, ConfigDict, ValidationError, conlist from typing import List @@ -10,58 +7,23 @@ import os import sys import json import pytz +import pickle import numpy as np import pandas as pd import joblib -from datetime import datetime, timedelta +from urllib.parse import urlparse from datetime import datetime import dateutil.parser from google.cloud import storage +from features.user_history import FEATURE_COLUMNS app = Flask(__name__) logging.basicConfig(level=logging.INFO, stream=sys.stdout) -TRAIN_FEATURES = [ - "item_has_thumbnail", - "item_has_site_icon", +USER_HISTORY_PATH = 'user_features.pkl' +MODEL_PIPELINE_PATH = 'predict_read_pipeline-v002.pkl' - 'user_30d_interactions_author_count', - 'user_30d_interactions_site_count', - 'user_30d_interactions_subscription_count', - - 'user_30d_interactions_author_rate', - 'user_30d_interactions_site_rate', - 'user_30d_interactions_subscription_rate', - - 'global_30d_interactions_site_count', - 'global_30d_interactions_author_count', - 'global_30d_interactions_subscription_count', - - 'global_30d_interactions_site_rate', - 'global_30d_interactions_author_rate', - 'global_30d_interactions_subscription_rate' -] - -DB_PARAMS = { - 'dbname': os.getenv('DB_NAME') or 'omnivore', - 'user': os.getenv('DB_USER'), - 'password': os.getenv('DB_PASSWORD'), - 'host': os.getenv('DB_HOST') or 'localhost', - 'port': os.getenv('DB_PORT') or '5432' -} - -USER_FEATURES = { - "site": "user_30d_interactions_site", - "author": "user_30d_interactions_author", - "subscription": "user_30d_interactions_subscription", -} - -GLOBAL_FEATURES = { - "site": "global_30d_interactions_site", - "author": "global_30d_interactions_author", - "subscription": "global_30d_interactions_subscription", -} def download_from_gcs(bucket_name, gcs_path, destination_path): storage_client = storage.Client() @@ -70,94 +32,44 @@ def download_from_gcs(bucket_name, gcs_path, destination_path): blob.download_to_filename(destination_path) -def load_pipeline(): - bucket_name = os.getenv('GCS_BUCKET') - pipeline_gcs_path = os.getenv('PIPELINE_GCS_PATH') - download_from_gcs(bucket_name, pipeline_gcs_path, '/tmp/pipeline.pkl') - pipeline = joblib.load('/tmp/pipeline.pkl') +def load_pipeline(path): + pipeline = joblib.load(path) return pipeline -def load_pipeline_local(): - pipeline = joblib.load('predict_user_clicked_random_forest_pipeline-v001.pkl') - return pipeline +def load_tables_from_pickle(path): + with open(path, 'rb') as handle: + tables = pickle.load(handle) + return tables -def fetch_user_features(name, feature_name): - conn = psycopg2.connect(**DB_PARAMS) - cur = conn.cursor() - query = f"SELECT user_id, {name}, interactions, interaction_rate FROM {feature_name}" - - cur.execute(query) - data = cur.fetchall() - - cur.close() - conn.close() - columns = [ - "user_id", - name, - "interactions", - "interaction_rate" - ] - - rate_feature_name = f"{feature_name}_rate" - count_feature_name = f"{feature_name}_count" - - df_loaded = pd.DataFrame(data, columns=columns) - df_loaded = df_loaded.rename(columns={"interactions": count_feature_name}, errors="raise") - df_loaded = df_loaded.rename(columns={"interaction_rate": rate_feature_name}, errors="raise") - df_loaded[rate_feature_name] = df_loaded[rate_feature_name].fillna(0) - df_loaded[count_feature_name] = df_loaded[count_feature_name].fillna(0) - - return df_loaded - - -def fetch_global_features(name, feature_name): - conn = psycopg2.connect(**DB_PARAMS) - cur = conn.cursor() - query = f"SELECT {name}, interactions, interaction_rate FROM {feature_name}" - - cur.execute(query) - data = cur.fetchall() - - cur.close() - conn.close() - columns = [ - name, - "interactions", - "interaction_rate" - ] - - rate_feature_name = f"{feature_name}_rate" - count_feature_name = f"{feature_name}_count" - - df_loaded = pd.DataFrame(data, columns=columns) - df_loaded = df_loaded.rename(columns={"interactions": count_feature_name}, errors="raise") - df_loaded = df_loaded.rename(columns={"interaction_rate": rate_feature_name}, errors="raise") - df_loaded[rate_feature_name] = df_loaded[rate_feature_name].fillna(0) - df_loaded[count_feature_name] = df_loaded[count_feature_name].fillna(0) - - return df_loaded - - -def load_user_features(): +def load_user_features(path): result = {} - for view_name in USER_FEATURES.keys(): - key_name = USER_FEATURES[view_name] - result[key_name] = fetch_user_features(view_name, key_name) - app.logger.info(f"loaded {len(result[key_name])} features for {key_name}") + tables = load_tables_from_pickle(path) + for table_name in tables.keys(): + result[table_name] = tables[table_name].to_pandas() return result -def load_global_features(): +def dataframe_to_dict(df): result = {} - for view_name in GLOBAL_FEATURES.keys(): - key_name = GLOBAL_FEATURES[view_name] - result[key_name] = fetch_global_features(view_name, key_name) - app.logger.info(f"loaded {len(result[key_name])} features for {key_name}") + for index, row in df.iterrows(): + user_id = row['user_id'] + if user_id not in result: + result[user_id] = [] + result[user_id].append(row.to_dict()) return result +def merge_dicts(dict1, dict2): + for key, value in dict2.items(): + if key in dict1: + dict1[key].extend(value) + else: + dict1[key] = value + return dict1 + + def compute_score(user_id, item_features): interaction_score = compute_interaction_score(user_id, item_features) return { @@ -166,86 +78,55 @@ def compute_score(user_id, item_features): } -def compute_time_bonus_score(item_features): - saved_at = item_features['saved_at'] - current_time = datetime.now(pytz.utc) - time_diff_hours = (current_time - saved_at).total_seconds() / 3600 - max_diff_hours = 3 * 24 - if time_diff_hours >= max_diff_hours: - return 0.0 - else: - return max(0.0, min(1.0, 1 - (time_diff_hours / max_diff_hours))) - - def compute_interaction_score(user_id, item_features): - print('item_features', item_features) + original_url_host = urlparse(item_features.get('original_url')).netloc df_test = pd.DataFrame([{ 'user_id': user_id, 'author': item_features.get('author'), 'site': item_features.get('site'), 'subscription': item_features.get('subscription'), + 'original_url_host': original_url_host, 'item_has_thumbnail': 1 if item_features.get('has_thumbnail') else 0, "item_has_site_icon": 1 if item_features.get('has_site_icon') else 0, + + 'item_word_count': item_features.get('words_count'), + 'is_subscription': 1 if item_features.get('is_subscription') else 0, + 'is_newsletter': 1 if item_features.get('is_newsletter') else 0, + 'is_feed': 1 if item_features.get('is_feed') else 0, + 'days_since_subscribed': item_features.get('days_since_subscribed'), + 'subscription_count': item_features.get('subscription_count'), + 'subscription_auto_add_to_library': item_features.get('subscription_auto_add_to_library'), + 'subscription_fetch_content': item_features.get('subscription_fetch_content'), + + 'has_author': 1 if item_features.get('author') else 0, + 'inbox_folder': 1 if item_features.get('folder') == 'inbox' else 0, }]) - for name in USER_FEATURES.keys(): - feature_name = USER_FEATURES[name] - df_feature = user_features[feature_name] - df_test = df_test.merge(df_feature, on=['user_id', name], how='left') - df_test[f"{feature_name}_rate"] = df_test[f"{feature_name}_rate"].fillna(0) - df_test[f"{feature_name}_count"] = df_test[f"{feature_name}_count"].fillna(0) + for name, df in user_features.items(): + df = df[df['user_id'] == user_id] + if 'author' in name: + merge_keys = ['user_id', 'author'] + elif 'site' in name: + merge_keys = ['user_id', 'site'] + elif 'subscription' in name: + merge_keys = ['user_id', 'subscription'] + elif 'original_url_host' in name: + merge_keys = ['user_id', 'original_url_host'] + else: + print("skipping feature: ", name) + continue - for name in GLOBAL_FEATURES.keys(): - feature_name = GLOBAL_FEATURES[name] - df_feature = global_features[feature_name] - df_test = df_test.merge(df_feature, on=name, how='left') - df_test[f"{feature_name}_rate"] = df_test[f"{feature_name}_rate"].fillna(0) - df_test[f"{feature_name}_count"] = df_test[f"{feature_name}_count"].fillna(0) + df_test = pd.merge(df_test, df, on=merge_keys, how='left') + df_test = df_test.fillna(0) + df_predict = df_test[FEATURE_COLUMNS] - df_predict = df_test[TRAIN_FEATURES] - - # Print out the columns with values, so we can know how sparse our data is - #scored_columns = df_predict.columns[(df_predict.notnull() & (df_predict != 0)).any()].tolist() - #print("scored columns", scored_columns) interaction_score = pipeline.predict_proba(df_predict) + print('score', interaction_score, 'item_features', df_test[df_test != 0].stack()) return interaction_score[0][1] -def get_library_item(library_item_id): - conn = psycopg2.connect(**DB_PARAMS) - cur = conn.cursor() - query = """ - SELECT - li.title, - li.author, - li.saved_at, - li.site_name as site, - li.item_language as language, - li.subscription, - li.word_count, - li.directionality, - CASE WHEN li.thumbnail IS NOT NULL then 1 else 0 END as has_thumbnail, - CASE WHEN li.site_icon IS NOT NULL then 1 else 0 END as has_site_icon - FROM omnivore.library_item li - WHERE li.id = %s - """ - - cur.execute(query, (library_item_id,)) - - data = cur.fetchone() - columns = [desc[0] for desc in cur.description] - cur.close() - conn.close() - - if data: - item_dict = dict(zip(columns, data)) - return item_dict - else: - return None - - @app.route('/_ah/health', methods=['GET']) def ready(): return jsonify({'OK': 'yes'}), 200 @@ -254,29 +135,17 @@ def ready(): @app.route('/users//features', methods=['GET']) def get_user_features(user_id): result = {} + df_user = pd.DataFrame([{ + 'user_id': user_id, + }]) - for name in USER_FEATURES.keys(): - feature_name = USER_FEATURES[name] - rate_feature_name = f"{feature_name}_rate" - count_feature_name = f"{feature_name}_count" - df_feature = user_features[feature_name] - df_filtered = df_feature[df_feature['user_id'] == user_id] - if not df_filtered.empty: - rate = df_filtered[[name, rate_feature_name]].dropna().to_dict(orient='records') - count = df_filtered[[name, count_feature_name]].dropna().to_dict(orient='records') - result[feature_name] = { - 'rate': rate, - 'count': count - } + user_data = {} + for name, df in user_features.items(): + df = df[df['user_id'] == user_id] + df_dict = dataframe_to_dict(df) + user_data = merge_dicts(user_data, df_dict) - return jsonify(result), 200 - - -@app.route('/users//library_items//score', methods=['GET']) -def get_library_item_score(user_id, library_item_id): - item_features = get_library_item(library_item_id) - score = compute_score(user_id, item_features) - return jsonify({'score': score}) + return jsonify(user_data), 200 @app.route('/predict', methods=['POST']) @@ -287,7 +156,6 @@ def predict(): user_id = data.get('user_id') item_features = data.get('item_features') - item_features['saved_at'] = dateutil.parser.isoparse(item_features['saved_at']) if user_id is None: return jsonify({'error': 'Missing user_id'}), 400 @@ -316,7 +184,6 @@ def batch(): print('key": ', key) print('item: ', item) library_item_id = item['library_item_id'] - item['saved_at'] = dateutil.parser.isoparse(item['saved_at']) result[library_item_id] = compute_score(user_id, item) return jsonify(result) @@ -325,14 +192,15 @@ def batch(): return jsonify({'error': str(e)}), 500 -if os.getenv('LOAD_LOCAL_MODEL'): - pipeline = load_pipeline_local() -else: - pipeline = load_pipeline() +if os.getenv('LOAD_LOCAL_MODEL') != None: + gcs_bucket_name = os.getenv('GCS_BUCKET') + download_from_gcs(gcs_bucket_name, f'data/features/user_features.pkl', USER_HISTORY_PATH) + download_from_gcs(gcs_bucket_name, f'data/models/predict_read_pipeline-v002.pkl', MODEL_PIPELINE_PATH) -user_features = load_user_features() -global_features = load_global_features() +pipeline = load_pipeline(MODEL_PIPELINE_PATH) +user_features = load_user_features(USER_HISTORY_PATH) +print('loaded pipeline and user_features', pipeline, user_features) if __name__ == '__main__': app.run(debug=True, port=5000) \ No newline at end of file diff --git a/ml/digest-score/features.py b/ml/digest-score/features.py new file mode 100644 index 000000000..47766c244 --- /dev/null +++ b/ml/digest-score/features.py @@ -0,0 +1,31 @@ +import psycopg2 +import numpy as np +import pandas as pd +from sqlalchemy import create_engine, text +from datetime import datetime, timedelta + +import os +from io import BytesIO +import tempfile + +import pyarrow as pa +import pyarrow.parquet as pq +from google.cloud import storage + +from features.extract import extract_and_upload_raw_data +from features.user_history import generate_and_upload_user_history + + + +def main(): + execution_date = os.getenv('EXECUTION_DATE') + num_days_history = os.getenv('NUM_DAYS_HISTORY') + gcs_bucket_name = os.getenv('GCS_BUCKET') + + extract_and_upload_raw_data(execution_date, num_days_history, gcs_bucket_name) + generate_and_upload_user_history(execution_date, gcs_bucket_name) + + print("done") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ml/digest-score/features/__init__.py b/ml/digest-score/features/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ml/digest-score/features/extract.py b/ml/digest-score/features/extract.py new file mode 100644 index 000000000..58e70ecf8 --- /dev/null +++ b/ml/digest-score/features/extract.py @@ -0,0 +1,109 @@ +# extract and upload raw data used for feature generation + +import psycopg2 +import numpy as np +import pandas as pd +from sqlalchemy import create_engine, text +from datetime import datetime, timedelta + +import os +from io import BytesIO +import tempfile + +import pyarrow as pa +import pyarrow.parquet as pq +from google.cloud import storage + +DB_PARAMS = { + 'dbname': os.getenv('DB_NAME') or 'omnivore', + 'user': os.getenv('DB_USER'), + 'password': os.getenv('DB_PASSWORD'), + 'host': os.getenv('DB_HOST') or 'localhost', + 'port': os.getenv('DB_PORT') or '5432' +} + +def extract_host(url): + try: + return urlparse(url).netloc + except Exception as e: + return None + +def fetch_raw_data(date_str, num_days_history): + end_date = pd.to_datetime(date_str) + start_date = end_date - timedelta(days=num_days_history) + start_date_str = start_date.strftime('%Y-%m-%d 00:00:00') + end_date_str = end_date.strftime('%Y-%m-%d 23:59:59') + + conn_str = f"postgresql://{DB_PARAMS['user']}:{DB_PARAMS['password']}@{DB_PARAMS['host']}:{DB_PARAMS['port']}/{DB_PARAMS['dbname']}" + # conn_str = f"postgresql://{DB_PARAMS['host']}:{DB_PARAMS['port']}/{DB_PARAMS['dbname']}" + engine = create_engine(conn_str) + + query = text(""" + SELECT + li.id as library_item_id, + li.user_id, + li.created_at, + li.archived_at, + li.deleted_at, + CASE WHEN li.folder = 'inbox' then 1 else 0 END as inbox_folder, + li.item_type, + li.item_language AS language, + li.content_reader, + li.word_count as item_word_count, + CASE WHEN li.thumbnail IS NOT NULL then 1 else 0 END as item_has_thumbnail, + CASE WHEN li.site_icon IS NOT NULL then 1 else 0 END as item_has_site_icon, + li.original_url, + li.site_name AS site, + li.author, + li.subscription, + sub.type as subscription_type, + sub.created_at as subscription_start_date, + sub.count as subscription_count, + sub.auto_add_to_library as subscription_auto_add_to_library, + sub.fetch_content as subscription_fetch_content, + sub.folder as subscription_folder, + CASE WHEN li.read_at is not NULL then 1 else 0 END as user_clicked, + CASE WHEN li.reading_progress_bottom_percent > 10 THEN 1 ELSE 0 END AS user_read, + CASE WHEN li.reading_progress_bottom_percent > 50 THEN 1 ELSE 0 END AS user_long_read + FROM omnivore.library_item AS li + LEFT JOIN omnivore.subscriptions sub on li.subscription = sub.name AND sub.user_id = li.user_id + WHERE li.created_at >= :start_date AND li.created_at <= :end_date; + """) + + chunk_size = 100000 # Adjust based on available memory and performance needs + + with tempfile.TemporaryDirectory() as tmpdir: + parquet_files = [] + with engine.connect() as conn: + for i, chunk in enumerate(pd.read_sql(query, conn, params={'start_date': start_date_str, 'end_date': end_date_str}, chunksize=chunk_size)): + chunk['library_item_id'] = chunk['library_item_id'].astype(str) + chunk['user_id'] = chunk['user_id'].astype(str) + chunk['original_url_host'] = chunk['original_url'].apply(extract_host) + + parquet_file = os.path.join(tmpdir, f'chunk_{i}.parquet') + chunk.to_parquet(parquet_file) + parquet_files.append(parquet_file) + + concatenated_df = pd.concat([pd.read_parquet(file) for file in parquet_files], ignore_index=True) + + parquet_buffer = BytesIO() + table = pa.Table.from_pandas(concatenated_df) + pq.write_table(table, parquet_buffer) + parquet_buffer.seek(0) + + return parquet_buffer + + +def upload_raw_databuffer(feather_buffer, execution_date, gcs_bucket_name): + client = storage.Client() + bucket = client.bucket(gcs_bucket_name) + blob = bucket.blob(f'data/raw/library_items_{execution_date}.parquet') + blob.upload_from_file(feather_buffer, content_type='application/octet-stream') + + print("Data stored successfully.") + + +def extract_and_upload_raw_data(execution_date, num_days_history, gcs_bucket_name): + buffer = fetch_raw_data(execution_date, int(num_days_history)) + upload_raw_databuffer(buffer, execution_date, gcs_bucket_name) + diff --git a/ml/digest-score/features/user_history.py b/ml/digest-score/features/user_history.py new file mode 100644 index 000000000..3906e6170 --- /dev/null +++ b/ml/digest-score/features/user_history.py @@ -0,0 +1,235 @@ +# download raw user data, aggregate user history, and upload to GCS + +import psycopg2 +import numpy as np +import pandas as pd +from sqlalchemy import create_engine, text +from datetime import datetime, timedelta + +import os +from io import BytesIO +import tempfile + +import pickle +import pyarrow as pa +import pyarrow.parquet as pq +import pyarrow.feather as feather +from google.cloud import storage + +FEATURE_COLUMNS=[ + # targets + # 'user_clicked', 'user_read', 'user_long_read', + + # item attributes / user setup attributes + 'item_word_count','item_has_site_icon', 'is_subscription', + 'inbox_folder', 'has_author', + + # how the user has setup the subscription + 'is_newsletter', 'is_feed', 'days_since_subscribed', + 'subscription_count', 'subscription_auto_add_to_library', + 'subscription_fetch_content', + + # user/item interaction history + 'user_original_url_host_saved_count_week_1', + 'user_original_url_host_interaction_count_week_1', + 'user_original_url_host_rate_week_1', + 'user_original_url_host_proportion_week_1', + + 'user_original_url_host_saved_count_week_2', + 'user_original_url_host_interaction_count_week_2', + 'user_original_url_host_rate_week_2', + 'user_original_url_host_proportion_week_2', + 'user_original_url_host_saved_count_week_3', + 'user_original_url_host_interaction_count_week_3', + 'user_original_url_host_rate_week_3', + 'user_original_url_host_proportion_week_3', + 'user_original_url_host_saved_count_week_4', + 'user_original_url_host_interaction_count_week_4', + 'user_original_url_host_rate_week_4', + 'user_original_url_host_proportion_week_4', + + 'user_subscription_saved_count_week_1', + 'user_subscription_interaction_count_week_1', + 'user_subscription_rate_week_1', 'user_subscription_proportion_week_1', + 'user_site_saved_count_week_3', 'user_site_interaction_count_week_3', + 'user_site_rate_week_3', 'user_site_proportion_week_3', + 'user_site_saved_count_week_2', 'user_site_interaction_count_week_2', + 'user_site_rate_week_2', 'user_site_proportion_week_2', + 'user_subscription_saved_count_week_2', + 'user_subscription_interaction_count_week_2', + 'user_subscription_rate_week_2', 'user_subscription_proportion_week_2', + 'user_site_saved_count_week_1', 'user_site_interaction_count_week_1', + 'user_site_rate_week_1', 'user_site_proportion_week_1', + 'user_subscription_saved_count_week_3', + 'user_subscription_interaction_count_week_3', + 'user_subscription_rate_week_3', 'user_subscription_proportion_week_3', + 'user_author_saved_count_week_4', + 'user_author_interaction_count_week_4', 'user_author_rate_week_4', + 'user_author_proportion_week_4', 'user_author_saved_count_week_1', + 'user_author_interaction_count_week_1', 'user_author_rate_week_1', + 'user_author_proportion_week_1', 'user_site_saved_count_week_4', + 'user_site_interaction_count_week_4', 'user_site_rate_week_4', + 'user_site_proportion_week_4', 'user_author_saved_count_week_2', + 'user_author_interaction_count_week_2', 'user_author_rate_week_2', + 'user_author_proportion_week_2', 'user_author_saved_count_week_3', + 'user_author_interaction_count_week_3', 'user_author_rate_week_3', + 'user_author_proportion_week_3', 'user_subscription_saved_count_week_4', + 'user_subscription_interaction_count_week_4', + 'user_subscription_rate_week_4', 'user_subscription_proportion_week_4' +] + +def parquet_to_dataframe(file_path): + table = pq.read_table(file_path) + df = table.to_pandas() + return df + +def load_local_raw_library_items(): + local_file_path = '/Users/jacksonh/Downloads/data_raw_library_items_2024-03-01.parquet' + df = parquet_to_dataframe(local_file_path) + return df + +def load_tables_from_pickle(pickle_file): + with open(pickle_file, 'rb') as handle: + tables = pickle.load(handle) + return tables + + +def download_raw_library_items(execution_date, gcs_bucket_name): + local_file_path = 'raw_library_items.parquet' + + client = storage.Client() + bucket = client.bucket(gcs_bucket_name) + blob = bucket.blob(f'data/raw/library_items_{execution_date}.parquet') + blob.download_to_filename(local_file_path) + + df = parquet_to_dataframe(local_file_path) + + os.remove(local_file_path) + return df + + +def load_feather_files(feature_directory): + dataframes = {} + for file_name in os.listdir(feature_directory): + if file_name.endswith('.feather'): + file_path = os.path.join(feature_directory, file_name) + df_name = os.path.splitext(file_name)[0] # Use the file name (without extension) as key + table = feather.read_table(file_path) + dataframes[df_name] = table + return dataframes + + +# def save_tables_to_arrow_ipc(tables, output_file): +# with pa.OSFile(output_file, 'wb') as sink: +# with pa.ipc.new_stream(sink, tables[next(iter(tables))].schema) as writer: +# for name, table in tables.items(): +# print("NAME:", name, "TABLE", table) +# writer.write_table(table) + + +def save_tables_to_arrow_ipc_with_schemas(tables, output_file): + with pa.OSFile(output_file, 'wb') as sink: + with pa.ipc.new_stream(sink, pa.schema([])) as writer: + for name, table in tables.items(): + metadata = table.schema.metadata or {} + metadata = {**metadata, b'table_name': name.encode('utf-8')} + schema = table.schema.add_metadata(metadata) + print("NAME:", name, "TABLE", table) + writer.write_table(table.replace_schema_metadata(schema.metadata)) + + +def save_tables_to_pickle(tables, output_file): + with open(output_file, 'wb') as handle: + pickle.dump(tables, handle, protocol=pickle.HIGHEST_PROTOCOL) + + +def upload_to_gcs(bucket_name, source_file_name, destination_blob_name): + client = storage.Client() + bucket = client.bucket(bucket_name) + blob = bucket.blob(destination_blob_name) + blob.upload_from_filename(source_file_name) + print(f'File {source_file_name} uploaded to {destination_blob_name} in bucket {bucket_name}.') + + +def generate_and_upload_user_history(execution_date, gcs_bucket_name): + df = download_raw_library_items(execution_date, gcs_bucket_name) + # df = load_local_raw_library_items() + with tempfile.TemporaryDirectory() as tmpdir: + user_preferences = aggregate_user_preferences(df, tmpdir) + dataframes = load_feather_files(tmpdir) + filename = os.path.join(tmpdir, 'user_features.pkl') + save_tables_to_pickle(dataframes, filename) + files = load_tables_from_pickle(filename) + print("GENERATED FEATURE TABLES:", files.keys()) + for table in files.keys(): + print("TABLE: ", table, "LEN: ", len(files[table])) + upload_to_gcs(gcs_bucket_name, filename, f'data/features/user_features.pkl') + + + +def compute_dimension_aggregates(df, dimension, bucket_name): + # Compute initial aggregates to filter out items with less than 2 saved counts + initial_agg = df.groupby(['user_id', dimension]).size().reset_index(name='count') + filtered_df = df[df.set_index(['user_id', dimension]).index.isin(initial_agg[initial_agg['count'] >= 2].set_index(['user_id', dimension]).index)] + + agg = filtered_df.groupby(['user_id', dimension]).agg( + saved_count=(dimension, 'count'), + interaction_count=('user_clicked', 'sum') + ).reset_index() + + agg[f'user_{dimension}_rate_{bucket_name}'] = agg['interaction_count'] / agg['saved_count'] + agg[f'user_{dimension}_proportion_{bucket_name}'] = agg.groupby('user_id')['interaction_count'].transform(lambda x: x / x.sum()) + + agg = agg.rename(columns={ + 'saved_count': f'user_{dimension}_saved_count_{bucket_name}', + 'interaction_count': f'user_{dimension}_interaction_count_{bucket_name}' + }) + + return agg + +def calculate_and_save_aggregates(bucket_name, bucket_df, output_dir): + # Compute aggregates for each dimension + dimensions = ['author', 'site', 'original_url_host', 'subscription'] + for dimension in dimensions: + agg_df = compute_dimension_aggregates(bucket_df, dimension, bucket_name) + + # Save the aggregated DataFrame to a Feather file + filename = os.path.join(output_dir, f'user_{dimension}_{bucket_name}.feather') + save_aggregated_data(agg_df, filename) + print(f"Saved aggregated data for {dimension} in {bucket_name} to {filename}") + + +def save_aggregated_data(df, filename): + buffer = BytesIO() + df.to_feather(buffer) + buffer.seek(0) + + with open(filename, 'wb') as f: + f.write(buffer.getbuffer()) + + +def aggregate_user_preferences(df, output_dir): + # Convert 'created_at' to datetime + df['created_at'] = pd.to_datetime(df['created_at']) + + end_date = df['created_at'].max() + + # Define bucket ranges for the past four weeks + buckets = { + 'week_4': (end_date - timedelta(weeks=4), end_date - timedelta(weeks=3)), + 'week_3': (end_date - timedelta(weeks=3), end_date - timedelta(weeks=2)), + 'week_2': (end_date - timedelta(weeks=2), end_date - timedelta(weeks=1)), + 'week_1': (end_date - timedelta(weeks=1), end_date) + } + + # Calculate aggregates for each bucket and save to file + for bucket_name, (start_date, end_date) in buckets.items(): + bucket_df = df[(df['created_at'] >= start_date) & (df['created_at'] < end_date)] + calculate_and_save_aggregates(bucket_name, bucket_df, output_dir) + + + +def create_and_upload_user_history(execution_date, num_days_history, gcs_bucket_name): + buffer = download_raw_library_items(execution_date, gcs_bucket_name) + buffer = open_raw_library_items() + upload_raw_databuffer(buffer, execution_date, gcs_bucket_name) \ No newline at end of file diff --git a/ml/digest-score/requirements.txt b/ml/digest-score/requirements.txt index b0e2f280e..d62f7cf55 100644 --- a/ml/digest-score/requirements.txt +++ b/ml/digest-score/requirements.txt @@ -6,3 +6,5 @@ google-cloud-storage flask pydantic sklearn2pmml +sqlalchemy +pyarrow diff --git a/ml/digest-score/train.py b/ml/digest-score/train.py index 2789a2221..fdd4d0b52 100644 --- a/ml/digest-score/train.py +++ b/ml/digest-score/train.py @@ -1,20 +1,27 @@ -import psycopg2 import pandas as pd -import joblib -from datetime import datetime - import os -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler -from sklearn.ensemble import RandomForestClassifier -from sklearn2pmml import PMMLPipeline, sklearn2pmml -from sklearn.pipeline import Pipeline -from sklearn.metrics import accuracy_score, classification_report import numpy as np +from datetime import datetime, timedelta + +from sklearn.linear_model import SGDClassifier +from sklearn.ensemble import RandomForestClassifier, VotingClassifier + +from sklearn.preprocessing import StandardScaler + +from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, classification_report, confusion_matrix +from sklearn.utils import shuffle +from sklearn.model_selection import train_test_split +from sklearn2pmml import PMMLPipeline, sklearn2pmml from google.cloud import storage from google.cloud.exceptions import PreconditionFailed +import pickle +import pyarrow as pa +import pyarrow.parquet as pq +import pyarrow.feather as feather + +from features.user_history import FEATURE_COLUMNS DB_PARAMS = { 'dbname': os.getenv('DB_NAME') or 'omnivore', @@ -24,276 +31,184 @@ DB_PARAMS = { 'port': os.getenv('DB_PORT') or '5432' } - -TRAIN_FEATURES = [ - # "item_word_count", - "item_has_thumbnail", - "item_has_site_icon", - - 'user_30d_interactions_author_count', - 'user_30d_interactions_site_count', - 'user_30d_interactions_subscription_count', - - 'user_30d_interactions_author_rate', - 'user_30d_interactions_site_rate', - 'user_30d_interactions_subscription_rate', - - 'global_30d_interactions_site_count', - 'global_30d_interactions_author_count', - 'global_30d_interactions_subscription_count', - - 'global_30d_interactions_site_rate', - 'global_30d_interactions_author_rate', - 'global_30d_interactions_subscription_rate' -] - - -def fetch_data(sample_size): - # Connect to the PostgreSQL database - conn = psycopg2.connect(**DB_PARAMS) - cur = conn.cursor() - query = f""" - SELECT - user_id, - created_at, - item_folder, - item_type, - language, - content_reader, - directionality, - item_word_count, - item_has_thumbnail, - item_has_site_icon, - site, - author, - subscription, - item_subscription_type, - user_clicked, - user_read, - user_long_read - - FROM user_7d_activity LIMIT {sample_size} - """ - - cur.execute(query) - data = cur.fetchall() - - cur.close() - conn.close() - columns = [ - "user_id", - "created_at", - "item_folder", - "item_type", - "language", - "content_reader", - "directionality", - "item_word_count", - "item_has_thumbnail", - "item_has_site_icon", - "site", - "author", - "subscription", - "item_subscription_type", - "user_clicked", - "user_read", - "user_long_read", - ] - - df = pd.DataFrame(data, columns=columns) +def parquet_to_dataframe(file_path): + table = pq.read_table(file_path) + df = table.to_pandas() return df +def save_to_pickle(object, output_file): + with open(output_file, 'wb') as handle: + pickle.dump(object, handle, protocol=pickle.HIGHEST_PROTOCOL) -def add_user_features(df, name, feature_name): - conn = psycopg2.connect(**DB_PARAMS) - cur = conn.cursor() - query = f"SELECT user_id, {name}, interactions, interaction_rate FROM {feature_name}" +def load_tables_from_pickle(pickle_file): + with open(pickle_file, 'rb') as handle: + tables = pickle.load(handle) + return tables - cur.execute(query) - data = cur.fetchall() - - cur.close() - conn.close() - columns = [ - "user_id", - name, - "interactions", - "interaction_rate" - ] - - rate_feature_name = f"{feature_name}_rate" - count_feature_name = f"{feature_name}_count" - - df_loaded = pd.DataFrame(data, columns=columns) - df_loaded = df_loaded.rename(columns={"interactions": count_feature_name}, errors="raise") - df_loaded = df_loaded.rename(columns={"interaction_rate": rate_feature_name}, errors="raise") - df_merged = pd.merge(df, df_loaded[['user_id', name, rate_feature_name, count_feature_name]], on=['user_id',name], how='left') - - df_merged[rate_feature_name] = df_merged[rate_feature_name].fillna(0) - df_merged[count_feature_name] = df_merged[count_feature_name].fillna(0) - - return df_merged +def load_dataframes_from_pickle(pickle_file): + result = {} + tables = load_tables_from_pickle(pickle_file) + for table_name in tables.keys(): + result[table_name] = tables[table_name].to_pandas() + return result -def add_global_features(df, name, feature_name): - conn = psycopg2.connect(**DB_PARAMS) - cur = conn.cursor() - query = f"SELECT {name}, interactions, interaction_rate FROM {feature_name}" - - cur.execute(query) - data = cur.fetchall() - - cur.close() - conn.close() - columns = [ - name, - "interactions", - "interaction_rate" - ] - - rate_feature_name = f"{feature_name}_rate" - count_feature_name = f"{feature_name}_count" - - df_loaded = pd.DataFrame(data, columns=columns) - df_loaded = df_loaded.rename(columns={"interactions": count_feature_name}, errors="raise") - df_loaded = df_loaded.rename(columns={"interaction_rate": rate_feature_name}, errors="raise") - - df_merged = pd.merge(df, df_loaded[[name, count_feature_name, rate_feature_name]], on=name, how='left') - - df_merged[rate_feature_name] = df_merged[rate_feature_name].fillna(0) - df_merged[count_feature_name] = df_merged[count_feature_name].fillna(0) - return df_merged - - -def add_dummy_features(df): - known_folder_types = ['inbox', 'following'] - known_subscription_types = ['NEWSLETTER', 'RSS'] - # known_item_types = ['ARTICLE', 'BOOK', 'FILE', 'HIGHLIGHTS', 'IMAGE', 'PROFILE', 'TWEET', 'UNKNOWN','VIDEO','WEBSITE'] - #known_content_reader_types = ['WEB', 'PDF', 'EPUB'] - # known_directionality_types = ['LTR', 'RTL'] - - folder_dummies = pd.get_dummies(df['item_folder'], columns=known_subscription_types, prefix='item_folder') - subscription_type_dummies = pd.get_dummies(df['item_subscription_type'], columns=known_subscription_types, prefix='item_subscription_type') - - # item_type_dummies = pd.get_dummies(df['item_type'], columns=known_item_types, prefix='item_type') - # content_reader_dummies = pd.get_dummies(df['content_reader'], columns=known_content_reader_types, prefix='content_reader') - # directionality_dummies = pd.get_dummies(df['directionality'], columns=known_directionality_types, prefix='directionality') - # language_dummies = pd.get_dummies(df['language'], prefix='language') - - # if 'title_topic' in df.columns: - # title_topic_dummies = pd.get_dummies(df['title_topic'], prefix='title_topic') - - new_feature_names = list(subscription_type_dummies.columns) + list(folder_dummies.columns) - print("NEW FEATURE NAMES: ", new_feature_names) - # new_feature_names = list(item_type_dummies.columns) + list(content_reader_dummies.columns) + \ - # list(directionality_dummies.columns) + list(language_dummies.columns) - - # if 'title_topic' in df.columns: - # new_feature_names += list(title_topic_dummies.columns) - # , title_topic_dummies - return pd.concat([df, subscription_type_dummies, folder_dummies], axis=1), new_feature_names - - -def random_forest_predictor(df, feature_columns, user_interaction): - features = df[feature_columns] - - features = features.fillna(0) - target = df[user_interaction] - - X = features - y = target.values - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) - scaler = StandardScaler() - rf_classifier = RandomForestClassifier(n_estimators=50, max_depth=10, random_state=42) - - pipeline = PMMLPipeline([ - ("scaler", scaler), - ("classifier", rf_classifier) - ]) - pipeline.fit(X_train, y_train) - - y_pred = pipeline.predict(X_test) - - feature_importance = rf_classifier.feature_importances_ - - print("Feature Importance:") - for feature, importance in zip(feature_columns, feature_importance): - print(f"{feature}: {importance}") - - print("\nClassification Report:") - print(classification_report(y_test, y_pred)) - - return pipeline - - -def save_and_upload_model(pipeline, target_interaction_type, bucket_name): - pipeline_file_name = f'predict_{target_interaction_type}_random_forest_pipeline-v001.pkl' - joblib.dump(pipeline, pipeline_file_name) - - if bucket_name: - upload_to_gcs(bucket_name, pipeline_file_name, f'models/{pipeline_file_name}') - else: - print("No GCS credentials so i am not uploading") +def download_from_gcs(bucket_name, source_blob_name, destination_file_name): + client = storage.Client() + bucket = client.bucket(bucket_name) + blob = bucket.blob(source_blob_name) + blob.download_to_filename(destination_file_name) + print(f'Blob {source_blob_name} downloaded to {destination_file_name}.') def upload_to_gcs(bucket_name, source_file_name, destination_blob_name): - """Uploads a file to the bucket.""" storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) blob = bucket.blob(destination_blob_name) blob.upload_from_filename(source_file_name) - print(f"File {source_file_name} uploaded to {destination_blob_name}.") -def resample_data(df): - print("Initial distribution:\n", df['user_clicked'].value_counts()) +def load_and_sample_library_items_from_parquet(raw_file_path, sample_size): + df = parquet_to_dataframe(raw_file_path) + sampled_df = df.sample(frac=sample_size, random_state=42) + return sampled_df - # Separate the majority and minority classes - df_majority = df[df['user_clicked'] == False] - df_minority = df[df['user_clicked'] == True] - # Resample the minority class - df_minority_oversampled = df_minority.sample(n=len(df_majority), replace=True, random_state=42) +def merge_user_preference_data(sampled_raw_df, feature_dict): + # Start with the sampled raw DataFrame + merged_df = sampled_raw_df - # Combine the majority class with the oversampled minority class - df_balanced = pd.concat([df_majority, df_minority_oversampled]) + # Iterate through the files in the feature directory + for key in feature_dict.keys(): + user_preference_df = feature_dict[key] + + # Determine the dimension to join on + if 'author' in key: + merge_keys = ['user_id', 'author'] + elif 'site' in key: + merge_keys = ['user_id', 'site'] + elif 'subscription' in key: + merge_keys = ['user_id', 'subscription'] + elif 'original_url_host' in key: + merge_keys = ['user_id', 'original_url_host'] + else: + print("skipping feature: ", key) + continue # Skip files that don't match expected patterns + + # Merge with the current user preference DataFrame + merged_df = pd.merge(merged_df, user_preference_df, on=merge_keys, how='left') - # Shuffle the DataFrame to mix the classes - df_balanced = df_balanced.sample(frac=1, random_state=42).reset_index(drop=True) + # Optionally, fill NaNs after each merge step to avoid growing NaNs + merged_df = merged_df.fillna(0) + + return merged_df - # Check the new distribution - print("Balanced distribution:\n", df_balanced['user_clicked'].value_counts()) +def prepare_data(df): + df['created_at'] = pd.to_datetime(df['created_at']) + df['subscription_start_date'] = pd.to_datetime(df['subscription_start_date'], errors='coerce') - # Display the first few rows of the balanced DataFrame - print(df_balanced.head()) + df['is_subscription'] = df['subscription'].apply(lambda x: 1 if pd.notna(x) and x != '' else 0) + df['has_author'] = df['author'].apply(lambda x: 1 if pd.notna(x) and x != '' else 0) + + # Calculate the days since subscribed + df['days_since_subscribed'] = (df['created_at'] - df['subscription_start_date']).dt.days + + # Handle cases where subscription_start_date is NaT (Not a Time) or negative + df['days_since_subscribed'] = df['days_since_subscribed'].apply(lambda x: x if x >= 0 else 0) + df['days_since_subscribed'] = df['days_since_subscribed'].fillna(0).astype(int) + + df['is_feed'] = df['subscription_type'].apply(lambda x: 1 if x == 'RSS' else 0) + df['is_newsletter'] = df['subscription_type'].apply(lambda x: 1 if x == 'NEWSLETTER' else 0) + + df = df.dropna(subset=['user_clicked']) + + # Fill NaNs in other columns with 0 (if any remain) + df = df.fillna(0) + + X = df[FEATURE_COLUMNS] # .drop(columns=['user_id', 'user_clicked']) + Y = df['user_clicked'] + + return X, Y + +def train_random_forest_model(X, Y): + model = RandomForestClassifier( + class_weight={0: 1, 1: 10}, + n_estimators=10, + max_depth=10, + random_state=42 + ) + + scaler = StandardScaler() + X_scaled = scaler.fit_transform(X) + + X_train, X_test, Y_train, Y_test = train_test_split(X_scaled, Y, test_size=0.3, random_state=42) + + pipeline = PMMLPipeline([ + ("scaler", scaler), + ("classifier", model) + ]) + + pipeline.fit(X_train, Y_train) + + Y_pred = pipeline.predict(X_test) + print_classification_report(Y_test, Y_pred) + print_feature_importance(X, model) + + return pipeline + + +def print_feature_importance(X, rf): + # Get feature importances + importances = rf.feature_importances_ + + # Get the indices of the features sorted by importance + indices = np.argsort(importances)[::-1] + + # Print the feature ranking + print("Feature ranking:") + + for f in range(X.shape[1]): + print(f"{f + 1}. feature {indices[f]} ({importances[indices[f]]:.4f}) - {X.columns[indices[f]]}") + + + +def print_classification_report(Y_test, Y_pred): + report = classification_report(Y_test, Y_pred, target_names=['Not Clicked', 'Clicked'], output_dict=True) + print("Classification Report:") + print(f"Accuracy: {report['accuracy']:.4f}") + print(f"Precision (Not Clicked): {report['Not Clicked']['precision']:.4f}") + print(f"Recall (Not Clicked): {report['Not Clicked']['recall']:.4f}") + print(f"F1-Score (Not Clicked): {report['Not Clicked']['f1-score']:.4f}") + print(f"Precision (Clicked): {report['Clicked']['precision']:.4f}") + print(f"Recall (Clicked): {report['Clicked']['recall']:.4f}") + print(f"F1-Score (Clicked): {report['Clicked']['f1-score']:.4f}") - return df_balanced def main(): - sample_size = int(os.getenv('SAMPLE_SIZE')) or 1000 - num_days_history = int(os.getenv('NUM_DAYS_HISTORY')) or 21 - gcs_bucket = os.getenv('GCS_BUCKET') + execution_date = os.getenv('EXECUTION_DATE') + num_days_history = os.getenv('NUM_DAYS_HISTORY') + gcs_bucket_name = os.getenv('GCS_BUCKET') - print("about to fetch library data") - df = fetch_data(sample_size) - print("FETCHED", df) + raw_data_path = f'raw_library_items_${execution_date}.parquet' + user_history_path = 'features_user_features.pkl' + pipeline_path = 'predict_read_pipeline-v002.pkl' - df = add_user_features(df, 'author', 'user_30d_interactions_author') - df = add_user_features(df, 'site', 'user_30d_interactions_site') - df = add_user_features(df, 'subscription', 'user_30d_interactions_subscription') - df = add_global_features(df, 'site', 'global_30d_interactions_site') - df = add_global_features(df, 'author', 'global_30d_interactions_author') - df = add_global_features(df, 'subscription', 'global_30d_interactions_subscription') + download_from_gcs(gcs_bucket_name, f'data/features/user_features.pkl', user_history_path) + download_from_gcs(gcs_bucket_name, f'data/raw/library_items_{execution_date}.parquet', raw_data_path) - df = resample_data(df) - print("training RandomForest with number of library_items: ", len(df)) - pipeline = random_forest_predictor(df, TRAIN_FEATURES, 'user_clicked') + sampled_raw_df = load_and_sample_library_items_from_parquet(raw_data_path, 0.10) + user_history = load_dataframes_from_pickle(user_history_path) - print(f"uploading model and scaler to {gcs_bucket}") - save_and_upload_model(pipeline, 'user_clicked', gcs_bucket) + merged_df = merge_user_preference_data(sampled_raw_df, user_history) + + print("created merged data", merged_df.columns) + + X, Y = prepare_data(merged_df) + random_forest_pipeline = train_random_forest_model(X, Y) + save_to_pickle(random_forest_pipeline, pipeline_path) + upload_to_gcs(gcs_bucket_name, pipeline_path, f'data/models/{pipeline_path}') - print("done") if __name__ == "__main__": main() \ No newline at end of file diff --git a/packages/api/test/resolvers/article_saving_request.test.ts b/packages/api/test/resolvers/article_saving_request.test.ts index 9df8b57f5..c47099bd1 100644 --- a/packages/api/test/resolvers/article_saving_request.test.ts +++ b/packages/api/test/resolvers/article_saving_request.test.ts @@ -97,15 +97,15 @@ describe('ArticleSavingRequest API', () => { ).to.eql(ArticleSavingRequestStatus.Processing) }) - it('creates a library item in db', async () => { - const url = 'https://blog.omnivore.app/1' - await graphqlRequest( - createArticleSavingRequestMutation('https://blog.omnivore.app/1'), + it('returns an error if the url is invalid', async () => { + const res = await graphqlRequest( + createArticleSavingRequestMutation('invalid url'), authToken ).expect(200) - const item = await findLibraryItemByUrl(url, user.id) - expect(item?.readableContent).to.eql('Your link is being saved...') + expect(res.body.data.createArticleSavingRequest.errorCodes).to.eql([ + CreateArticleSavingRequestErrorCode.BadData, + ]) }) it('returns an error if the url is invalid', async () => { From 0e321143f65a71dab6b8621b98fe969200c853ac Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Thu, 20 Jun 2024 15:52:23 +0800 Subject: [PATCH 05/32] Clean up --- ml/digest-score/app.py | 10 +++++----- ml/digest-score/features/user_history.py | 13 ------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/ml/digest-score/app.py b/ml/digest-score/app.py index 501a417be..f9c428100 100644 --- a/ml/digest-score/app.py +++ b/ml/digest-score/app.py @@ -90,13 +90,13 @@ def compute_interaction_score(user_id, item_features): 'item_has_thumbnail': 1 if item_features.get('has_thumbnail') else 0, "item_has_site_icon": 1 if item_features.get('has_site_icon') else 0, - 'item_word_count': item_features.get('words_count'), + 'item_word_count': item_features.get('words_count'), 'is_subscription': 1 if item_features.get('is_subscription') else 0, - 'is_newsletter': 1 if item_features.get('is_newsletter') else 0, + 'is_newsletter': 1 if item_features.get('is_newsletter') else 0, 'is_feed': 1 if item_features.get('is_feed') else 0, - 'days_since_subscribed': item_features.get('days_since_subscribed'), - 'subscription_count': item_features.get('subscription_count'), - 'subscription_auto_add_to_library': item_features.get('subscription_auto_add_to_library'), + 'days_since_subscribed': item_features.get('days_since_subscribed'), + 'subscription_count': item_features.get('subscription_count'), + 'subscription_auto_add_to_library': item_features.get('subscription_auto_add_to_library'), 'subscription_fetch_content': item_features.get('subscription_fetch_content'), 'has_author': 1 if item_features.get('author') else 0, diff --git a/ml/digest-score/features/user_history.py b/ml/digest-score/features/user_history.py index 3906e6170..289ecc64d 100644 --- a/ml/digest-score/features/user_history.py +++ b/ml/digest-score/features/user_history.py @@ -83,10 +83,6 @@ def parquet_to_dataframe(file_path): df = table.to_pandas() return df -def load_local_raw_library_items(): - local_file_path = '/Users/jacksonh/Downloads/data_raw_library_items_2024-03-01.parquet' - df = parquet_to_dataframe(local_file_path) - return df def load_tables_from_pickle(pickle_file): with open(pickle_file, 'rb') as handle: @@ -119,14 +115,6 @@ def load_feather_files(feature_directory): return dataframes -# def save_tables_to_arrow_ipc(tables, output_file): -# with pa.OSFile(output_file, 'wb') as sink: -# with pa.ipc.new_stream(sink, tables[next(iter(tables))].schema) as writer: -# for name, table in tables.items(): -# print("NAME:", name, "TABLE", table) -# writer.write_table(table) - - def save_tables_to_arrow_ipc_with_schemas(tables, output_file): with pa.OSFile(output_file, 'wb') as sink: with pa.ipc.new_stream(sink, pa.schema([])) as writer: @@ -153,7 +141,6 @@ def upload_to_gcs(bucket_name, source_file_name, destination_blob_name): def generate_and_upload_user_history(execution_date, gcs_bucket_name): df = download_raw_library_items(execution_date, gcs_bucket_name) - # df = load_local_raw_library_items() with tempfile.TemporaryDirectory() as tmpdir: user_preferences = aggregate_user_preferences(df, tmpdir) dataframes = load_feather_files(tmpdir) From 49cce94b297e35c028786e0b5129daa218394f26 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 21 Jun 2024 09:18:33 +0800 Subject: [PATCH 06/32] Linting clean ups --- ml/digest-score/train.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/ml/digest-score/train.py b/ml/digest-score/train.py index fdd4d0b52..db8deff84 100644 --- a/ml/digest-score/train.py +++ b/ml/digest-score/train.py @@ -76,14 +76,10 @@ def load_and_sample_library_items_from_parquet(raw_file_path, sample_size): def merge_user_preference_data(sampled_raw_df, feature_dict): - # Start with the sampled raw DataFrame merged_df = sampled_raw_df - # Iterate through the files in the feature directory for key in feature_dict.keys(): user_preference_df = feature_dict[key] - - # Determine the dimension to join on if 'author' in key: merge_keys = ['user_id', 'author'] elif 'site' in key: @@ -95,13 +91,8 @@ def merge_user_preference_data(sampled_raw_df, feature_dict): else: print("skipping feature: ", key) continue # Skip files that don't match expected patterns - - # Merge with the current user preference DataFrame merged_df = pd.merge(merged_df, user_preference_df, on=merge_keys, how='left') - - # Optionally, fill NaNs after each merge step to avoid growing NaNs merged_df = merged_df.fillna(0) - return merged_df def prepare_data(df): @@ -119,7 +110,7 @@ def prepare_data(df): df['days_since_subscribed'] = df['days_since_subscribed'].fillna(0).astype(int) df['is_feed'] = df['subscription_type'].apply(lambda x: 1 if x == 'RSS' else 0) - df['is_newsletter'] = df['subscription_type'].apply(lambda x: 1 if x == 'NEWSLETTER' else 0) + df['is_newsletter'] = df['subscription_type'].apply(lambda x: 1 if x == 'NEWSLETTER' else 0) df = df.dropna(subset=['user_clicked']) From d7801eb20213f8e4871b782506f21ec47955fcba Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 21 Jun 2024 09:22:14 +0800 Subject: [PATCH 07/32] Remove test change --- .../test/resolvers/article_saving_request.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/test/resolvers/article_saving_request.test.ts b/packages/api/test/resolvers/article_saving_request.test.ts index c47099bd1..9df8b57f5 100644 --- a/packages/api/test/resolvers/article_saving_request.test.ts +++ b/packages/api/test/resolvers/article_saving_request.test.ts @@ -97,15 +97,15 @@ describe('ArticleSavingRequest API', () => { ).to.eql(ArticleSavingRequestStatus.Processing) }) - it('returns an error if the url is invalid', async () => { - const res = await graphqlRequest( - createArticleSavingRequestMutation('invalid url'), + it('creates a library item in db', async () => { + const url = 'https://blog.omnivore.app/1' + await graphqlRequest( + createArticleSavingRequestMutation('https://blog.omnivore.app/1'), authToken ).expect(200) - expect(res.body.data.createArticleSavingRequest.errorCodes).to.eql([ - CreateArticleSavingRequestErrorCode.BadData, - ]) + const item = await findLibraryItemByUrl(url, user.id) + expect(item?.readableContent).to.eql('Your link is being saved...') }) it('returns an error if the url is invalid', async () => { From f33e5323c66e789d1834f3abfcfd64e5756325f5 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 21 Jun 2024 10:56:20 +0800 Subject: [PATCH 08/32] Start to use nav return in most places to return to the previous library section --- packages/web/components/nav-containers/home.tsx | 2 +- .../patterns/LibraryCards/LibraryGridCard.tsx | 2 +- .../patterns/LibraryCards/LibraryListCard.tsx | 2 +- packages/web/components/templates/SettingsLayout.tsx | 2 +- .../templates/article/PdfArticleContainer.tsx | 11 +++++------ packages/web/pages/[username]/[slug]/index.tsx | 4 ++-- packages/web/pages/_app.tsx | 4 ++-- packages/web/pages/api/client/auth.ts | 2 +- packages/web/pages/index.tsx | 7 ++++++- packages/web/pages/tools/bulk.tsx | 2 +- packages/web/pages/tools/import/file.tsx | 2 +- packages/web/pages/tools/import/matter-archive.tsx | 2 +- 12 files changed, 23 insertions(+), 19 deletions(-) diff --git a/packages/web/components/nav-containers/home.tsx b/packages/web/components/nav-containers/home.tsx index 1174b3334..4ff835594 100644 --- a/packages/web/components/nav-containers/home.tsx +++ b/packages/web/components/nav-containers/home.tsx @@ -40,7 +40,7 @@ export function HomeContainer(): JSX.Element { }, [viewerData]) useEffect(() => { - window.sessionStorage.setItem('nav-return', router.asPath) + window.localStorage.setItem('nav-return', router.asPath) }, [router.asPath]) return ( diff --git a/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx b/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx index 857816954..8bba60f0b 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryGridCard.tsx @@ -95,7 +95,7 @@ export function LibraryGridCard(props: LinkedItemCardProps): JSX.Element { props.setIsChecked(props.item.id, !props.isChecked) return } - window.sessionStorage.setItem('nav-return', router.asPath) + window.localStorage.setItem('nav-return', router.asPath) if (event.metaKey || event.ctrlKey) { window.open( `/${props.viewer.profile.username}/${props.item.slug}`, diff --git a/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx b/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx index a7d277230..5fb5d00ab 100644 --- a/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx +++ b/packages/web/components/patterns/LibraryCards/LibraryListCard.tsx @@ -100,7 +100,7 @@ export function LibraryListCard(props: LinkedItemCardProps): JSX.Element { props.setIsChecked(props.item.id, !props.isChecked) return } - window.sessionStorage.setItem('nav-return', router.asPath) + window.localStorage.setItem('nav-return', router.asPath) if (event.metaKey || event.ctrlKey) { window.open( `/${props.viewer.profile.username}/${props.item.slug}`, diff --git a/packages/web/components/templates/SettingsLayout.tsx b/packages/web/components/templates/SettingsLayout.tsx index a6716af9d..f1eb850cf 100644 --- a/packages/web/components/templates/SettingsLayout.tsx +++ b/packages/web/components/templates/SettingsLayout.tsx @@ -32,7 +32,7 @@ const ReturnButton = (): JSX.Element => { }, }} > - + (null) const [notebookKey, setNotebookKey] = useState(uuidv4()) const [noteTarget, setNoteTarget] = useState(undefined) - const [noteTargetPageIndex, setNoteTargetPageIndex] = useState< - number | undefined - >(undefined) + const [noteTargetPageIndex, setNoteTargetPageIndex] = + useState(undefined) const highlightsRef = useRef([]) const annotationOmnivoreId = (annotation: Annotation): string | undefined => { @@ -449,16 +448,16 @@ export default function PdfArticleContainer( document.dispatchEvent(new Event('openOriginalArticle')) break case 'u': - const navReturn = window.sessionStorage.getItem('nav-return') + const navReturn = window.localStorage.getItem('nav-return') if (navReturn) { window.location.assign(navReturn) return } const query = window.sessionStorage.getItem('q') if (query) { - window.location.assign(`/home?${query}`) + window.location.assign(`/l/home?${query}`) } else { - window.location.replace(`/home`) + window.location.replace(`/l/home`) } break case 'e': diff --git a/packages/web/pages/[username]/[slug]/index.tsx b/packages/web/pages/[username]/[slug]/index.tsx index 45d2f94c4..559f4f8d0 100644 --- a/packages/web/pages/[username]/[slug]/index.tsx +++ b/packages/web/pages/[username]/[slug]/index.tsx @@ -89,7 +89,7 @@ export default function Home(): JSX.Element { // return // } // } - const navReturn = window.sessionStorage.getItem('nav-return') + const navReturn = window.localStorage.getItem('nav-return') if (navReturn) { router.push(navReturn) return @@ -303,7 +303,7 @@ export default function Home(): JSX.Element { name: 'Return to library', shortcut: ['u'], perform: () => { - const navReturn = window.sessionStorage.getItem('nav-return') + const navReturn = window.localStorage.getItem('nav-return') if (navReturn) { router.push(navReturn) return diff --git a/packages/web/pages/_app.tsx b/packages/web/pages/_app.tsx index e81880576..86f82fcce 100644 --- a/packages/web/pages/_app.tsx +++ b/packages/web/pages/_app.tsx @@ -43,12 +43,12 @@ const generateActions = (router: NextRouter) => { shortcut: ['g', 'h'], keywords: 'go home', perform: () => { - const navReturn = window.sessionStorage.getItem('nav-return') + const navReturn = window.localStorage.getItem('nav-return') if (navReturn) { router.push(navReturn) return } - router?.push('/home') + router?.push('/l/home') }, }, { diff --git a/packages/web/pages/api/client/auth.ts b/packages/web/pages/api/client/auth.ts index ee1d2dc01..e63d81767 100644 --- a/packages/web/pages/api/client/auth.ts +++ b/packages/web/pages/api/client/auth.ts @@ -29,7 +29,7 @@ const requestHandler = (req: NextApiRequest, res: NextApiResponse): void => { }) } else { res.writeHead(302, { - Location: '/home', + Location: '/l/home', }) } diff --git a/packages/web/pages/index.tsx b/packages/web/pages/index.tsx index 199614dc2..51b2b3b14 100644 --- a/packages/web/pages/index.tsx +++ b/packages/web/pages/index.tsx @@ -9,7 +9,12 @@ export default function LandingPage(): JSX.Element { const { viewerData, isLoading } = useGetViewerQuery() if (!isLoading && router.isReady && viewerData?.me) { - router.push('/home') + const navReturn = window.localStorage.getItem('nav-return') + if (navReturn) { + router.push(navReturn) + } else { + router.push('/l/home') + } return <> } else if (isLoading || !router.isReady) { return ( diff --git a/packages/web/pages/tools/bulk.tsx b/packages/web/pages/tools/bulk.tsx index 24efa499f..f1142d20b 100644 --- a/packages/web/pages/tools/bulk.tsx +++ b/packages/web/pages/tools/bulk.tsx @@ -202,7 +202,7 @@ export default function BulkPerformer(): JSX.Element { )} - {/* */} - {props.homeItem.canArchive && ( )} {props.homeItem.canShare && ( - )} From b7ebb091d9f4667ca673b78b7ad8a77da830ab1c Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 21 Jun 2024 11:06:21 +0800 Subject: [PATCH 10/32] Prevent shift when unsubscribe button is presented on hover cards --- packages/web/components/nav-containers/home.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/web/components/nav-containers/home.tsx b/packages/web/components/nav-containers/home.tsx index b5069f2bb..2930b0a19 100644 --- a/packages/web/components/nav-containers/home.tsx +++ b/packages/web/components/nav-containers/home.tsx @@ -853,7 +853,7 @@ const SubscriptionSourceHoverContent = ( {props.source.icon && } {subscription && subscription.status == 'ACTIVE' && ( )} From 07e78c9c60a41ff91354207a34b1e70c35d3ecad Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 21 Jun 2024 11:21:53 +0800 Subject: [PATCH 11/32] Show ten top picks at once --- packages/web/components/nav-containers/home.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/components/nav-containers/home.tsx b/packages/web/components/nav-containers/home.tsx index 2930b0a19..8c4a3d6b6 100644 --- a/packages/web/components/nav-containers/home.tsx +++ b/packages/web/components/nav-containers/home.tsx @@ -273,7 +273,7 @@ const TopPicksHomeSection = (props: HomeSectionProps): JSX.Element => { ( Date: Fri, 21 Jun 2024 11:22:06 +0800 Subject: [PATCH 12/32] Make the display settings respect theme --- packages/web/components/elements/ModalPrimitives.tsx | 1 + packages/web/components/elements/TickedRangeSlider.tsx | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/web/components/elements/ModalPrimitives.tsx b/packages/web/components/elements/ModalPrimitives.tsx index 465a97c9c..2b8a8deeb 100644 --- a/packages/web/components/elements/ModalPrimitives.tsx +++ b/packages/web/components/elements/ModalPrimitives.tsx @@ -35,6 +35,7 @@ const Modal = styled(Content, { export const ModalContent = styled(Modal, { top: '50%', left: '50%', + bg: '$readerBg', transform: 'translate(-50%, -50%)', width: '90vw', maxWidth: '450px', diff --git a/packages/web/components/elements/TickedRangeSlider.tsx b/packages/web/components/elements/TickedRangeSlider.tsx index da7696569..b4119e1ce 100644 --- a/packages/web/components/elements/TickedRangeSlider.tsx +++ b/packages/web/components/elements/TickedRangeSlider.tsx @@ -22,14 +22,14 @@ const StyledSlider = styled(Slider, { height: '8px', width: '225px', borderRadius: '10px', - backgroundColor: '#F2F2F2', + backgroundColor: '$thTextSubtle2', }, '.SliderThumb': { display: 'block', - width: '20px', - height: '20px', + width: '15px', + height: '15px', borderRadius: '50%', - border: '4px solid white', + border: '2px solid $thTextSubtle2', backgroundColor: '#FFD234', boxShadow: '0px 0px 20px rgba(19, 56, 77, 0.2)', }, From 5a63af25f96eca700cd282963da2719ae1f7e2ea Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 21 Jun 2024 11:42:36 +0800 Subject: [PATCH 13/32] Alter omnivore_admin role to prevent omnivore_admin to be inherited by app_user or omnivore_user --- .../0183.do.alter_omnivore_admin_role.sql | 36 +++++++++++++++++++ .../0183.undo.alter_omnivore_admin_role.sql | 31 ++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100755 packages/db/migrations/0183.do.alter_omnivore_admin_role.sql create mode 100755 packages/db/migrations/0183.undo.alter_omnivore_admin_role.sql diff --git a/packages/db/migrations/0183.do.alter_omnivore_admin_role.sql b/packages/db/migrations/0183.do.alter_omnivore_admin_role.sql new file mode 100755 index 000000000..5a87699eb --- /dev/null +++ b/packages/db/migrations/0183.do.alter_omnivore_admin_role.sql @@ -0,0 +1,36 @@ +-- Type: DO +-- Name: alter_omnivore_admin_role +-- Description: Alter omnivore_admin role to prevent omnivore_admin to be inherited by app_user or omnivore_user + +BEGIN; + +DROP POLICY user_admin_policy ON omnivore.user; + +REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA omnivore from omnivore_admin; +REVOKE ALL PRIVILEGES ON SCHEMA omnivore from omnivore_admin; + +DROP OWNED BY omnivore_admin; + +DROP ROLE omnivore_admin; + +CREATE ROLE omnivore_admin; + +GRANT USAGE ON SCHEMA omnivore TO omnivore_admin; + +ALTER ROLE omnivore_user NOINHERIT; -- This is to prevent omnivore_user from inheriting omnivore_admin role + +GRANT omnivore_admin TO omnivore_user; -- This is to allow app_user to set omnivore_admin role + +GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.user TO omnivore_admin; +CREATE POLICY user_admin_policy on omnivore.user + FOR ALL + TO omnivore_admin + USING (true); + +GRANT SELECT, INSERT, UPDATE, DELETE ON omnivore.library_item TO omnivore_admin; +CREATE POLICY library_item_admin_policy ON omnivore.library_item + FOR ALL + TO omnivore_admin + USING (true); + +COMMIT; diff --git a/packages/db/migrations/0183.undo.alter_omnivore_admin_role.sql b/packages/db/migrations/0183.undo.alter_omnivore_admin_role.sql new file mode 100755 index 000000000..0b8c5fa6e --- /dev/null +++ b/packages/db/migrations/0183.undo.alter_omnivore_admin_role.sql @@ -0,0 +1,31 @@ +-- Type: UNDO +-- Name: alter_omnivore_admin_role +-- Description: Alter omnivore_admin role to prevent omnivore_admin to be inherited by app_user or omnivore_user + +BEGIN; + +DROP POLICY library_item_admin_policy ON omnivore.library_item; +REVOKE SELECT, INSERT, UPDATE, DELETE ON omnivore.library_item FROM omnivore_admin; + +DROP POLICY user_admin_policy ON omnivore.user; +REVOKE SELECT, INSERT, UPDATE, DELETE ON omnivore.user FROM omnivore_admin; + +DROP OWNED BY omnivore_admin; + +DROP ROLE omnivore_admin; + +ALTER ROLE omnivore_user INHERIT; + +CREATE ROLE omnivore_admin; + +GRANT omnivore_admin TO app_user; + +GRANT ALL PRIVILEGES ON SCHEMA omnivore TO omnivore_admin; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA omnivore TO omnivore_admin; + +CREATE POLICY user_admin_policy on omnivore.user + FOR ALL + TO omnivore_admin + USING (true); + +COMMIT; From 977a1a90e52e034ef5d4995b6e1f0f90578e8e58 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 21 Jun 2024 12:00:53 +0800 Subject: [PATCH 14/32] Skip scoring RSS items for now --- packages/api/src/jobs/score_library_item.ts | 57 +++++++++++++++++---- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/packages/api/src/jobs/score_library_item.ts b/packages/api/src/jobs/score_library_item.ts index 8f084422b..2e89f9955 100644 --- a/packages/api/src/jobs/score_library_item.ts +++ b/packages/api/src/jobs/score_library_item.ts @@ -1,5 +1,10 @@ -import { findLibraryItemById } from '../services/library_item' +import { SubscriptionType } from '../entity/subscription' +import { + findLibraryItemById, + updateLibraryItem, +} from '../services/library_item' import { Feature, scoreClient } from '../services/score' +import { findSubscriptionsByNames } from '../services/subscriptions' import { enqueueUpdateHomeJob } from '../utils/createTask' import { lanaugeToCode } from '../utils/helpers' import { logger } from '../utils/logger' @@ -31,6 +36,8 @@ export const scoreLibraryItem = async ( 'author', 'itemLanguage', 'wordCount', + 'subscription', + 'publishedAt', ], }) if (!libraryItem) { @@ -38,6 +45,26 @@ export const scoreLibraryItem = async ( return } + let subscription + if (libraryItem.subscription) { + const subscriptions = await findSubscriptionsByNames(userId, [ + libraryItem.subscription, + ]) + + if (subscriptions.length) { + subscription = subscriptions[0] + + if (subscription.type === SubscriptionType.Rss) { + logger.info('Skipping scoring for RSS subscription', { + userId, + libraryItemId, + }) + + return + } + } + } + const itemFeatures = { [libraryItem.id]: { library_item_id: libraryItem.id, @@ -53,7 +80,15 @@ export const scoreLibraryItem = async ( language: lanaugeToCode(libraryItem.itemLanguage || 'English'), word_count: libraryItem.wordCount, published_at: libraryItem.publishedAt, - subscription: libraryItem.subscription, + subscription: subscription?.name, + inbox_folder: libraryItem.folder === 'inbox', + is_feed: subscription?.type === SubscriptionType.Rss, + is_newsletter: subscription?.type === SubscriptionType.Newsletter, + is_subscription: !!subscription, + item_word_count: libraryItem.wordCount, + subscription_auto_add_to_library: subscription?.autoAddToLibrary, + subscription_fetch_content: subscription?.fetchContent, + subscription_count: 0, } as Feature, } @@ -69,15 +104,15 @@ export const scoreLibraryItem = async ( throw new Error('Failed to score library item') } - // await updateLibraryItem( - // libraryItem.id, - // { - // score, - // }, - // userId, - // undefined, - // true - // ) + await updateLibraryItem( + libraryItem.id, + { + score, + }, + userId, + undefined, + true + ) logger.info('Library item scored', data) try { From 14a375db83ae2f3520038ed70a0054c1b57a7716 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Fri, 21 Jun 2024 12:01:34 +0800 Subject: [PATCH 15/32] use real scoring service --- packages/api/src/services/score.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts index 25c78f5a0..8752e4b4a 100644 --- a/packages/api/src/services/score.ts +++ b/packages/api/src/services/score.ts @@ -79,4 +79,4 @@ class ScoreClientImpl implements ScoreClient { } } -export const scoreClient = new StubScoreClientImpl() +export const scoreClient = new ScoreClientImpl() From 80bded24db276a6ec695808c69a1cb49b43d36e4 Mon Sep 17 00:00:00 2001 From: Jackson Harper Date: Fri, 21 Jun 2024 12:36:40 +0800 Subject: [PATCH 16/32] Rename web container files --- .../components/nav-containers/highlights.tsx | 319 ------- .../web/components/nav-containers/home.tsx | 887 ------------------ packages/web/pages/l/[section].tsx | 4 +- 3 files changed, 2 insertions(+), 1208 deletions(-) delete mode 100644 packages/web/components/nav-containers/highlights.tsx delete mode 100644 packages/web/components/nav-containers/home.tsx diff --git a/packages/web/components/nav-containers/highlights.tsx b/packages/web/components/nav-containers/highlights.tsx deleted file mode 100644 index ef1358249..000000000 --- a/packages/web/components/nav-containers/highlights.tsx +++ /dev/null @@ -1,319 +0,0 @@ -import { NavigationLayout } from '../templates/NavigationLayout' -import { Box, HStack, VStack } from '../elements/LayoutPrimitives' -import { useFetchMore } from '../../lib/hooks/useFetchMoreScroll' -import { useCallback, useMemo, useState } from 'react' -import { useGetHighlights } from '../../lib/networking/queries/useGetHighlights' -import { Highlight } from '../../lib/networking/fragments/highlightFragment' -import { NextRouter, useRouter } from 'next/router' -import { - UserBasicData, - useGetViewerQuery, -} from '../../lib/networking/queries/useGetViewerQuery' -import { SetHighlightLabelsModalPresenter } from '../templates/article/SetLabelsModalPresenter' -import { TrashIcon } from '../elements/icons/TrashIcon' -import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers' -import { ConfirmationModal } from '../patterns/ConfirmationModal' -import { deleteHighlightMutation } from '../../lib/networking/mutations/deleteHighlightMutation' -import { LabelChip } from '../elements/LabelChip' -import ReactMarkdown from 'react-markdown' -import remarkGfm from 'remark-gfm' -import { timeAgo } from '../patterns/LibraryCards/LibraryCardStyles' -import { HighlightHoverActions } from '../patterns/HighlightHoverActions' -import { - autoUpdate, - offset, - size, - useFloating, - useHover, - useInteractions, -} from '@floating-ui/react' -import { highlightColor } from '../../lib/themeUpdater' - -import { HighlightViewNote } from '../patterns/HighlightNotes' -import { theme } from '../tokens/stitches.config' - -const PAGE_SIZE = 10 - -export function HighlightsContainer(): JSX.Element { - const router = useRouter() - const viewer = useGetViewerQuery() - const [showFilterMenu, setShowFilterMenu] = useState(false) - const [_, setShowAddLinkModal] = useState(false) - - const { isLoading, setSize, size, data, mutate } = useGetHighlights({ - first: PAGE_SIZE, - }) - - const hasMore = useMemo(() => { - if (!data) { - return false - } - return data[data.length - 1].highlights.pageInfo.hasNextPage - }, [data]) - - const handleFetchMore = useCallback(() => { - if (isLoading || !hasMore) { - return - } - setSize(size + 1) - }, [isLoading, hasMore, setSize, size]) - - useFetchMore(handleFetchMore) - - const highlights = useMemo(() => { - if (!data) { - return [] - } - return data.flatMap((res) => res.highlights.edges.map((edge) => edge.node)) - }, [data]) - - return ( - - {highlights.map((highlight) => { - return ( - viewer.viewerData?.me && ( - - ) - ) - })} - - ) -} - -type HighlightCardProps = { - highlight: Highlight - viewer: UserBasicData - router: NextRouter - mutate: () => void -} - -type HighlightAnnotationProps = { - highlight: Highlight -} - -function HighlightAnnotation({ - highlight, -}: HighlightAnnotationProps): JSX.Element { - const [noteMode, setNoteMode] = useState<'edit' | 'preview'>('preview') - const [annotation, setAnnotation] = useState(highlight.annotation) - - return ( - { - setAnnotation(highlight.annotation) - }} - /> - ) -} - -function HighlightCard(props: HighlightCardProps): JSX.Element { - const [isOpen, setIsOpen] = useState(false) - const [showConfirmDeleteHighlightId, setShowConfirmDeleteHighlightId] = - useState(undefined) - const [labelsTarget, setLabelsTarget] = useState( - undefined - ) - - const viewInReader = useCallback( - (highlightId: string) => { - const router = props.router - const viewer = props.viewer - const item = props.highlight.libraryItem - - if (!router || !router.isReady || !viewer || !item) { - showErrorToast('Error navigating to highlight') - return - } - - router.push( - { - pathname: '/[username]/[slug]', - query: { - username: viewer.profile.username, - slug: item.slug, - }, - hash: highlightId, - }, - `${viewer.profile.username}/${item.slug}#${highlightId}`, - { - scroll: false, - } - ) - }, - [props.highlight.libraryItem, props.viewer, props.router] - ) - - const { refs, floatingStyles, context } = useFloating({ - open: isOpen, - onOpenChange: setIsOpen, - middleware: [ - offset({ - mainAxis: -25, - }), - size(), - ], - placement: 'top-end', - whileElementsMounted: autoUpdate, - }) - - const hover = useHover(context) - - const { getReferenceProps, getFloatingProps } = useInteractions([hover]) - - return ( - - - - - - - {timeAgo(props.highlight.updatedAt)} - - {props.highlight.quote && ( - - {props.highlight.quote} - - )} - - {props.highlight.labels && ( - - {props.highlight.labels.map((label) => { - return ( - - ) - })} - - )} - - {props.highlight.libraryItem?.title} - - - {props.highlight.libraryItem?.author} - - {showConfirmDeleteHighlightId && ( - { - ;(async () => { - const highlightId = showConfirmDeleteHighlightId - const success = await deleteHighlightMutation( - props.highlight.libraryItem?.id || '', - showConfirmDeleteHighlightId - ) - props.mutate() - if (success) { - showSuccessToast('Highlight deleted.', { - position: 'bottom-right', - }) - const event = new CustomEvent('deleteHighlightbyId', { - detail: highlightId, - }) - document.dispatchEvent(event) - } else { - showErrorToast('Error deleting highlight', { - position: 'bottom-right', - }) - } - })() - setShowConfirmDeleteHighlightId(undefined) - }} - onOpenChange={() => setShowConfirmDeleteHighlightId(undefined)} - icon={ - - } - /> - )} - {labelsTarget && ( - { - // Don't actually need to do something here - console.log('update highlight: ', highlight) - }} - onOpenChange={() => { - props.mutate() - setLabelsTarget(undefined) - }} - /> - )} - - ) -} diff --git a/packages/web/components/nav-containers/home.tsx b/packages/web/components/nav-containers/home.tsx deleted file mode 100644 index 8c4a3d6b6..000000000 --- a/packages/web/components/nav-containers/home.tsx +++ /dev/null @@ -1,887 +0,0 @@ -import * as HoverCard from '@radix-ui/react-hover-card' -import { styled } from '@stitches/react' -import { useRouter } from 'next/router' -import { useCallback, useEffect, useMemo, useReducer, useState } from 'react' -import { Button } from '../elements/Button' -import { AddToLibraryActionIcon } from '../elements/icons/home/AddToLibraryActionIcon' -import { ArchiveActionIcon } from '../elements/icons/home/ArchiveActionIcon' -import { CommentActionIcon } from '../elements/icons/home/CommentActionIcon' -import { RemoveActionIcon } from '../elements/icons/home/RemoveActionIcon' -import { ShareActionIcon } from '../elements/icons/home/ShareActionIcon' -import Pagination from '../elements/Pagination' -import { timeAgo } from '../patterns/LibraryCards/LibraryCardStyles' -import { theme } from '../tokens/stitches.config' -import { useApplyLocalTheme } from '../../lib/hooks/useApplyLocalTheme' -import { useGetHiddenHomeSection } from '../../lib/networking/queries/useGetHiddenHomeSection' -import { - HomeItem, - HomeItemSource, - HomeItemSourceType, - HomeSection, - useGetHomeItems, -} from '../../lib/networking/queries/useGetHome' -import { - SubscriptionType, - useGetSubscriptionsQuery, -} from '../../lib/networking/queries/useGetSubscriptionsQuery' -import { Box, HStack, SpanBox, VStack } from '../elements/LayoutPrimitives' -import { Toaster } from 'react-hot-toast' -import { useGetViewerQuery } from '../../lib/networking/queries/useGetViewerQuery' - -export function HomeContainer(): JSX.Element { - const router = useRouter() - const homeData = useGetHomeItems() - const { viewerData } = useGetViewerQuery() - - useApplyLocalTheme() - - const viewerUsername = useMemo(() => { - return viewerData?.me?.profile.username - }, [viewerData]) - - useEffect(() => { - window.localStorage.setItem('nav-return', router.asPath) - }, [router.asPath]) - - return ( - - - - {homeData.sections?.map((homeSection, idx) => { - if (homeSection.items.length < 1) { - console.log('empty home section: ', homeSection) - return - } - switch (homeSection.layout) { - case 'just_added': - return ( - - ) - case 'top_picks': - return ( - - ) - case 'quick_links': - return ( - - ) - case 'hidden': - return ( - - ) - default: - console.log('unknown home section: ', homeSection) - return - } - })} - - - ) -} - -type HomeSectionProps = { - homeSection: HomeSection - viewerUsername: string | undefined -} - -const JustAddedHomeSection = (props: HomeSectionProps): JSX.Element => { - const router = useRouter() - return ( - - - - {props.homeSection.title} - - - - - - - {props.homeSection.items.map((homeItem) => { - return - })} - - - ) -} - -const TopPicksHomeSection = (props: HomeSectionProps): JSX.Element => { - const listReducer = ( - state: HomeItem[], - action: { - type: string - itemId?: string - items?: HomeItem[] - } - ) => { - console.log('handling action: ', action) - switch (action.type) { - case 'RESET': - return action.items ?? [] - case 'REMOVE_ITEM': - return state.filter((item) => item.id !== action.itemId) - default: - throw new Error() - } - } - - const [items, dispatchList] = useReducer(listReducer, []) - - function handleDelete(item: HomeItem) { - dispatchList({ - type: 'REMOVE_ITEM', - itemId: item.id, - }) - } - - useEffect(() => { - dispatchList({ - type: 'RESET', - items: props.homeSection.items, - }) - }, [props]) - - return ( - - {items.length > 0 && ( - - {props.homeSection.title} - - )} - - ( - - )} - /> - - ) -} - -const QuickLinksHomeSection = (props: HomeSectionProps): JSX.Element => { - return ( - - - {props.homeSection.title} - - - ( - - )} - /> - - ) -} - -const HiddenHomeSection = (props: HomeSectionProps): JSX.Element => { - const [isHidden, setIsHidden] = useState(true) - return ( - - setIsHidden(!isHidden)} - > - - {props.homeSection.title} - - - {isHidden ? 'Show' : 'Hide'} - - - - {isHidden ? <> : } - - ) -} - -const HiddenHomeSectionView = (): JSX.Element => { - const hiddenSectionData = useGetHiddenHomeSection() - - if (hiddenSectionData.error) { - return Error loading hidden section - } - - if (hiddenSectionData.isValidating) { - return Loading... - } - - if (!hiddenSectionData.section) { - return No hidden section data - } - - return ( - - {hiddenSectionData.section.items.map((homeItem) => { - return - })} - - ) -} - -const CoverImage = styled('img', { - objectFit: 'cover', -}) - -type HomeItemViewProps = { - homeItem: HomeItem - viewerUsername?: string | undefined -} - -const TimeAgo = (props: HomeItemViewProps): JSX.Element => { - return ( - - {timeAgo(props.homeItem.date)} - - ) -} - -const Title = (props: HomeItemViewProps): JSX.Element => { - return ( - - {props.homeItem.title} - - ) -} - -const TitleSmall = (props: HomeItemViewProps): JSX.Element => { - return ( - - {props.homeItem.title} - - ) -} - -type PreviewContentProps = { - previewContent?: string - maxLines?: string -} - -const PreviewContent = (props: PreviewContentProps): JSX.Element => { - return ( - - {props.previewContent ?? ''} - - ) -} - -const JustAddedItemView = (props: HomeItemViewProps): JSX.Element => { - const router = useRouter() - - return ( - { - const path = `/${props.viewerUsername ?? 'me'}/${props.homeItem.slug}` - if (event.metaKey || event.ctrlKey) { - window.open(path, '_blank') - } else { - router.push(path) - } - }} - > - - - - - - - - - - ) -} - -type TopPicksItemViewProps = { - dispatchList: (args: { type: string; itemId?: string }) => void -} - -const TopPicksItemView = ( - props: HomeItemViewProps & TopPicksItemViewProps -): JSX.Element => { - const router = useRouter() - return ( - { - const path = `/${props.viewerUsername ?? 'me'}/${props.homeItem.slug}` - if (event.metaKey || event.ctrlKey) { - window.open(path, '_blank') - } else { - router.push(path) - } - }} - alignment="start" - > - - - - - - - - - {props.homeItem.thumbnail && ( - - )} - - <PreviewContent - previewContent={props.homeItem.previewContent} - maxLines="6" - /> - </Box> - <SpanBox css={{ px: '20px' }}></SpanBox> - <HStack css={{ gap: '10px', my: '15px', px: '20px' }}> - {props.homeItem.canSave && ( - <Button - style="homeAction" - onClick={(event) => { - event.preventDefault() - event.stopPropagation() - - props.dispatchList({ - type: 'REMOVE_ITEM', - itemId: props.homeItem.id, - }) - }} - > - <AddToLibraryActionIcon - color={theme.colors.homeActionIcons.toString()} - /> - </Button> - )} - {props.homeItem.canArchive && ( - <Button - style="homeAction" - onClick={(event) => { - event.preventDefault() - event.stopPropagation() - - props.dispatchList({ - type: 'REMOVE_ITEM', - itemId: props.homeItem.id, - }) - }} - > - <ArchiveActionIcon - color={theme.colors.homeActionIcons.toString()} - /> - </Button> - )} - {props.homeItem.canDelete && ( - <Button - style="homeAction" - onClick={(event) => { - event.preventDefault() - event.stopPropagation() - - props.dispatchList({ - type: 'REMOVE_ITEM', - itemId: props.homeItem.id, - }) - }} - > - <RemoveActionIcon color={theme.colors.homeActionIcons.toString()} /> - </Button> - )} - {props.homeItem.canShare && ( - <Button - style="homeAction" - onClick={(event) => { - event.preventDefault() - event.stopPropagation() - }} - > - <ShareActionIcon color={theme.colors.homeActionIcons.toString()} /> - </Button> - )} - </HStack> - <Box - css={{ mt: '15px', width: '100%', height: '1px', bg: '$homeDivider' }} - /> - </VStack> - ) -} - -const QuickLinkHomeItemView = (props: HomeItemViewProps): JSX.Element => { - const router = useRouter() - - return ( - <VStack - css={{ - mt: '10px', - width: '100%', - px: '10px', - py: '10px', - gap: '5px', - borderRadius: '5px', - '&:hover': { - bg: '#007AFF10', - cursor: 'pointer', - }, - '&:hover .title-text': { - textDecoration: 'underline', - }, - }} - onClick={(event) => { - const path = `/${props.viewerUsername ?? 'me'}/${props.homeItem.slug}` - if (event.metaKey || event.ctrlKey) { - window.open(path, '_blank') - } else { - router.push(path) - } - }} - > - <HStack - distribution="start" - alignment="center" - css={{ width: '100%', gap: '5px', lineHeight: '1' }} - > - <SourceInfo homeItem={props.homeItem} subtle={true} /> - - <SpanBox css={{ ml: 'auto', flexShrink: '0' }}> - <TimeAgo homeItem={props.homeItem} /> - </SpanBox> - </HStack> - <Title homeItem={props.homeItem} /> - <PreviewContent - previewContent={props.homeItem.previewContent} - maxLines="2" - /> - </VStack> - ) -} - -const SiteIconSmall = styled('img', { - width: '16px', - height: '16px', - borderRadius: '100px', -}) - -const SiteIconLarge = styled('img', { - width: '25px', - height: '25px', - borderRadius: '100px', -}) - -type SourceInfoProps = { - subtle?: boolean -} - -const SourceInfo = (props: HomeItemViewProps & SourceInfoProps) => ( - <HoverCard.Root> - <HoverCard.Trigger asChild> - <HStack - distribution="start" - alignment="center" - css={{ - gap: '8px', - height: '16px', - cursor: 'pointer', - flex: '1', - overflow: 'hidden', - whiteSpace: 'nowrap', - textOverflow: 'ellipsis', - }} - > - {props.homeItem.source.icon && ( - <SiteIconSmall src={props.homeItem.source.icon} /> - )} - <HStack - css={{ - lineHeight: '1', - fontFamily: '$inter', - fontWeight: '500', - fontSize: props.subtle ? '12px' : '13px', - color: props.subtle ? '$homeTextSubtle' : '$homeTextSource', - textDecoration: 'underline', - }} - > - {props.homeItem.source.name} - </HStack> - </HStack> - </HoverCard.Trigger> - <HoverCard.Portal> - <HoverCard.Content sideOffset={5} style={{ zIndex: 5 }}> - <SubscriptionSourceHoverContent source={props.homeItem.source} /> - <HoverCard.Arrow fill={theme.colors.thBackground2.toString()} /> - </HoverCard.Content> - </HoverCard.Portal> - </HoverCard.Root> -) - -type SourceHoverContentProps = { - source: HomeItemSource -} - -const SubscriptionSourceHoverContent = ( - props: SourceHoverContentProps -): JSX.Element => { - const mapSourceType = ( - sourceType: HomeItemSourceType - ): SubscriptionType | undefined => { - switch (sourceType) { - case 'RSS': - case 'NEWSLETTER': - return sourceType as SubscriptionType - default: - return undefined - } - } - const { subscriptions, isValidating } = useGetSubscriptionsQuery( - mapSourceType(props.source.type) - ) - const subscription = useMemo(() => { - if (props.source.id && subscriptions) { - return subscriptions.find((sub) => sub.id == props.source.id) - } - return undefined - }, [subscriptions]) - - return ( - <VStack - alignment="start" - distribution="start" - css={{ - width: '380px', - height: '200px', - bg: '$thBackground2', - borderRadius: '10px', - padding: '15px', - gap: '10px', - boxShadow: theme.shadows.cardBoxShadow.toString(), - }} - > - <HStack - distribution="start" - alignment="center" - css={{ width: '100%', gap: '10px', height: '35px' }} - > - {props.source.icon && <SiteIconLarge src={props.source.icon} />} - <SpanBox - css={{ - fontFamily: '$inter', - fontWeight: '500', - fontSize: '14px', - }} - > - {props.source.name} - </SpanBox> - <SpanBox css={{ ml: 'auto', minWidth: '100px' }}> - {subscription && subscription.status == 'ACTIVE' && ( - <Button style="ctaSubtle" css={{ fontSize: '12px' }}> - Unsubscribe - </Button> - )} - </SpanBox> - </HStack> - <SpanBox - css={{ - fontFamily: '$inter', - fontSize: '13px', - color: '$homeTextBody', - }} - > - {subscription ? <>{subscription.description}</> : <></>} - </SpanBox> - </VStack> - ) -} diff --git a/packages/web/pages/l/[section].tsx b/packages/web/pages/l/[section].tsx index fc943e05b..782e7e14f 100644 --- a/packages/web/pages/l/[section].tsx +++ b/packages/web/pages/l/[section].tsx @@ -4,10 +4,10 @@ import { NavigationLayout, NavigationSection, } from '../../components/templates/NavigationLayout' -import { HomeContainer } from '../../components/nav-containers/home' +import { HomeContainer } from '../../components/nav-containers/HomeContainer' import { LibraryContainer } from '../../components/templates/library/LibraryContainer' import { useMemo } from 'react' -import { HighlightsContainer } from '../../components/nav-containers/highlights' +import { HighlightsContainer } from '../../components/nav-containers/HighlightsContainer' export default function Home(): JSX.Element { const router = useRouter() From 5c85257cdbb8d1eaf869125accf260c463771197 Mon Sep 17 00:00:00 2001 From: Jackson Harper <jacksonh@gmail.com> Date: Fri, 21 Jun 2024 13:20:27 +0800 Subject: [PATCH 17/32] Implement actions on the home view --- .../nav-containers/HighlightsContainer.tsx | 319 ++++++ .../nav-containers/HomeContainer.tsx | 920 ++++++++++++++++++ .../web/components/tokens/stitches.config.ts | 2 +- .../web/lib/hooks/useLibraryItemActions.tsx | 69 ++ .../mutations/moveToLibraryMutation.ts | 41 + 5 files changed, 1350 insertions(+), 1 deletion(-) create mode 100644 packages/web/components/nav-containers/HighlightsContainer.tsx create mode 100644 packages/web/components/nav-containers/HomeContainer.tsx create mode 100644 packages/web/lib/hooks/useLibraryItemActions.tsx create mode 100644 packages/web/lib/networking/mutations/moveToLibraryMutation.ts diff --git a/packages/web/components/nav-containers/HighlightsContainer.tsx b/packages/web/components/nav-containers/HighlightsContainer.tsx new file mode 100644 index 000000000..ef1358249 --- /dev/null +++ b/packages/web/components/nav-containers/HighlightsContainer.tsx @@ -0,0 +1,319 @@ +import { NavigationLayout } from '../templates/NavigationLayout' +import { Box, HStack, VStack } from '../elements/LayoutPrimitives' +import { useFetchMore } from '../../lib/hooks/useFetchMoreScroll' +import { useCallback, useMemo, useState } from 'react' +import { useGetHighlights } from '../../lib/networking/queries/useGetHighlights' +import { Highlight } from '../../lib/networking/fragments/highlightFragment' +import { NextRouter, useRouter } from 'next/router' +import { + UserBasicData, + useGetViewerQuery, +} from '../../lib/networking/queries/useGetViewerQuery' +import { SetHighlightLabelsModalPresenter } from '../templates/article/SetLabelsModalPresenter' +import { TrashIcon } from '../elements/icons/TrashIcon' +import { showErrorToast, showSuccessToast } from '../../lib/toastHelpers' +import { ConfirmationModal } from '../patterns/ConfirmationModal' +import { deleteHighlightMutation } from '../../lib/networking/mutations/deleteHighlightMutation' +import { LabelChip } from '../elements/LabelChip' +import ReactMarkdown from 'react-markdown' +import remarkGfm from 'remark-gfm' +import { timeAgo } from '../patterns/LibraryCards/LibraryCardStyles' +import { HighlightHoverActions } from '../patterns/HighlightHoverActions' +import { + autoUpdate, + offset, + size, + useFloating, + useHover, + useInteractions, +} from '@floating-ui/react' +import { highlightColor } from '../../lib/themeUpdater' + +import { HighlightViewNote } from '../patterns/HighlightNotes' +import { theme } from '../tokens/stitches.config' + +const PAGE_SIZE = 10 + +export function HighlightsContainer(): JSX.Element { + const router = useRouter() + const viewer = useGetViewerQuery() + const [showFilterMenu, setShowFilterMenu] = useState(false) + const [_, setShowAddLinkModal] = useState(false) + + const { isLoading, setSize, size, data, mutate } = useGetHighlights({ + first: PAGE_SIZE, + }) + + const hasMore = useMemo(() => { + if (!data) { + return false + } + return data[data.length - 1].highlights.pageInfo.hasNextPage + }, [data]) + + const handleFetchMore = useCallback(() => { + if (isLoading || !hasMore) { + return + } + setSize(size + 1) + }, [isLoading, hasMore, setSize, size]) + + useFetchMore(handleFetchMore) + + const highlights = useMemo(() => { + if (!data) { + return [] + } + return data.flatMap((res) => res.highlights.edges.map((edge) => edge.node)) + }, [data]) + + return ( + <VStack + css={{ + maxWidth: '70%', + padding: '20px', + margin: '30px 50px 0 0', + }} + > + {highlights.map((highlight) => { + return ( + viewer.viewerData?.me && ( + <HighlightCard + key={highlight.id} + highlight={highlight} + viewer={viewer.viewerData.me} + router={router} + mutate={mutate} + /> + ) + ) + })} + </VStack> + ) +} + +type HighlightCardProps = { + highlight: Highlight + viewer: UserBasicData + router: NextRouter + mutate: () => void +} + +type HighlightAnnotationProps = { + highlight: Highlight +} + +function HighlightAnnotation({ + highlight, +}: HighlightAnnotationProps): JSX.Element { + const [noteMode, setNoteMode] = useState<'edit' | 'preview'>('preview') + const [annotation, setAnnotation] = useState(highlight.annotation) + + return ( + <HighlightViewNote + targetId={highlight.id} + text={annotation} + placeHolder="Add notes to this highlight..." + highlight={highlight} + mode={noteMode} + setEditMode={setNoteMode} + updateHighlight={(highlight) => { + setAnnotation(highlight.annotation) + }} + /> + ) +} + +function HighlightCard(props: HighlightCardProps): JSX.Element { + const [isOpen, setIsOpen] = useState(false) + const [showConfirmDeleteHighlightId, setShowConfirmDeleteHighlightId] = + useState<undefined | string>(undefined) + const [labelsTarget, setLabelsTarget] = useState<Highlight | undefined>( + undefined + ) + + const viewInReader = useCallback( + (highlightId: string) => { + const router = props.router + const viewer = props.viewer + const item = props.highlight.libraryItem + + if (!router || !router.isReady || !viewer || !item) { + showErrorToast('Error navigating to highlight') + return + } + + router.push( + { + pathname: '/[username]/[slug]', + query: { + username: viewer.profile.username, + slug: item.slug, + }, + hash: highlightId, + }, + `${viewer.profile.username}/${item.slug}#${highlightId}`, + { + scroll: false, + } + ) + }, + [props.highlight.libraryItem, props.viewer, props.router] + ) + + const { refs, floatingStyles, context } = useFloating({ + open: isOpen, + onOpenChange: setIsOpen, + middleware: [ + offset({ + mainAxis: -25, + }), + size(), + ], + placement: 'top-end', + whileElementsMounted: autoUpdate, + }) + + const hover = useHover(context) + + const { getReferenceProps, getFloatingProps } = useInteractions([hover]) + + return ( + <VStack + ref={refs.setReference} + {...getReferenceProps()} + css={{ + width: '100%', + fontFamily: '$inter', + padding: '20px', + marginBottom: '20px', + bg: '$thBackground2', + borderRadius: '8px', + cursor: 'pointer', + }} + > + <Box + ref={refs.setFloating} + style={floatingStyles} + {...getFloatingProps()} + > + <HighlightHoverActions + viewer={props.viewer} + highlight={props.highlight} + isHovered={isOpen ?? false} + viewInReader={viewInReader} + setLabelsTarget={setLabelsTarget} + setShowConfirmDeleteHighlightId={setShowConfirmDeleteHighlightId} + /> + </Box> + <Box + css={{ + width: '30px', + height: '5px', + backgroundColor: highlightColor(props.highlight.color), + borderRadius: '2px', + }} + /> + <Box + css={{ + color: '$thText', + fontSize: '11px', + marginTop: '10px', + fontWeight: 300, + }} + > + {timeAgo(props.highlight.updatedAt)} + </Box> + {props.highlight.quote && ( + <ReactMarkdown remarkPlugins={[remarkGfm]}> + {props.highlight.quote} + </ReactMarkdown> + )} + <HighlightAnnotation highlight={props.highlight} /> + {props.highlight.labels && ( + <HStack + css={{ + marginBottom: '10px', + }} + > + {props.highlight.labels.map((label) => { + return ( + <LabelChip key={label.id} color={label.color} text={label.name} /> + ) + })} + </HStack> + )} + <Box + css={{ + color: '$thText', + fontSize: '12px', + lineHeight: '20px', + fontWeight: 300, + marginBottom: '10px', + }} + > + {props.highlight.libraryItem?.title} + </Box> + <Box + css={{ + color: '$grayText', + fontSize: '12px', + lineHeight: '20px', + fontWeight: 300, + }} + > + {props.highlight.libraryItem?.author} + </Box> + {showConfirmDeleteHighlightId && ( + <ConfirmationModal + message={'Are you sure you want to delete this highlight?'} + onAccept={() => { + ;(async () => { + const highlightId = showConfirmDeleteHighlightId + const success = await deleteHighlightMutation( + props.highlight.libraryItem?.id || '', + showConfirmDeleteHighlightId + ) + props.mutate() + if (success) { + showSuccessToast('Highlight deleted.', { + position: 'bottom-right', + }) + const event = new CustomEvent('deleteHighlightbyId', { + detail: highlightId, + }) + document.dispatchEvent(event) + } else { + showErrorToast('Error deleting highlight', { + position: 'bottom-right', + }) + } + })() + setShowConfirmDeleteHighlightId(undefined) + }} + onOpenChange={() => setShowConfirmDeleteHighlightId(undefined)} + icon={ + <TrashIcon + size={40} + color={theme.colors.grayTextContrast.toString()} + /> + } + /> + )} + {labelsTarget && ( + <SetHighlightLabelsModalPresenter + highlight={labelsTarget} + highlightId={labelsTarget.id} + onUpdate={(highlight) => { + // Don't actually need to do something here + console.log('update highlight: ', highlight) + }} + onOpenChange={() => { + props.mutate() + setLabelsTarget(undefined) + }} + /> + )} + </VStack> + ) +} diff --git a/packages/web/components/nav-containers/HomeContainer.tsx b/packages/web/components/nav-containers/HomeContainer.tsx new file mode 100644 index 000000000..59ff4e84a --- /dev/null +++ b/packages/web/components/nav-containers/HomeContainer.tsx @@ -0,0 +1,920 @@ +import * as HoverCard from '@radix-ui/react-hover-card' +import { styled } from '@stitches/react' +import { useRouter } from 'next/router' +import { useCallback, useEffect, useMemo, useReducer, useState } from 'react' +import { Button } from '../elements/Button' +import { AddToLibraryActionIcon } from '../elements/icons/home/AddToLibraryActionIcon' +import { ArchiveActionIcon } from '../elements/icons/home/ArchiveActionIcon' +import { CommentActionIcon } from '../elements/icons/home/CommentActionIcon' +import { RemoveActionIcon } from '../elements/icons/home/RemoveActionIcon' +import { ShareActionIcon } from '../elements/icons/home/ShareActionIcon' +import Pagination from '../elements/Pagination' +import { timeAgo } from '../patterns/LibraryCards/LibraryCardStyles' +import { theme } from '../tokens/stitches.config' +import { useApplyLocalTheme } from '../../lib/hooks/useApplyLocalTheme' +import { useGetHiddenHomeSection } from '../../lib/networking/queries/useGetHiddenHomeSection' +import { + HomeItem, + HomeItemSource, + HomeItemSourceType, + HomeSection, + useGetHomeItems, +} from '../../lib/networking/queries/useGetHome' +import { + SubscriptionType, + useGetSubscriptionsQuery, +} from '../../lib/networking/queries/useGetSubscriptionsQuery' +import { Box, HStack, SpanBox, VStack } from '../elements/LayoutPrimitives' +import { Toaster } from 'react-hot-toast' +import { useGetViewerQuery } from '../../lib/networking/queries/useGetViewerQuery' +import useLibraryItemActions from '../../lib/hooks/useLibraryItemActions' + +export function HomeContainer(): JSX.Element { + const router = useRouter() + const homeData = useGetHomeItems() + const { viewerData } = useGetViewerQuery() + + useApplyLocalTheme() + + const viewerUsername = useMemo(() => { + return viewerData?.me?.profile.username + }, [viewerData]) + + useEffect(() => { + window.localStorage.setItem('nav-return', router.asPath) + }, [router.asPath]) + + return ( + <VStack + distribution="start" + alignment="center" + css={{ + width: '100%', + bg: '$readerBg', + pt: '45px', + minHeight: '100vh', + minWidth: '320px', + '@mdDown': { + pt: '0px', + mt: '80px', + }, + }} + > + <Toaster /> + <VStack + distribution="start" + css={{ + width: '680px', + gap: '50px', + minHeight: '100vh', + '@mdDown': { + gap: '40px', + width: '100%', + }, + }} + > + {homeData.sections?.map((homeSection, idx) => { + if (homeSection.items.length < 1) { + console.log('empty home section: ', homeSection) + return <SpanBox key={`section-${idx}`}></SpanBox> + } + switch (homeSection.layout) { + case 'just_added': + return ( + <JustAddedHomeSection + key={`section-${idx}`} + homeSection={homeSection} + viewerUsername={viewerUsername} + /> + ) + case 'top_picks': + return ( + <TopPicksHomeSection + key={`section-${idx}`} + homeSection={homeSection} + viewerUsername={viewerUsername} + /> + ) + case 'quick_links': + return ( + <QuickLinksHomeSection + key={`section-${idx}`} + homeSection={homeSection} + viewerUsername={viewerUsername} + /> + ) + case 'hidden': + return ( + <HiddenHomeSection + key={`section-${idx}`} + homeSection={homeSection} + viewerUsername={viewerUsername} + /> + ) + default: + console.log('unknown home section: ', homeSection) + return <SpanBox key={`section-${idx}`}></SpanBox> + } + })} + </VStack> + </VStack> + ) +} + +type HomeSectionProps = { + homeSection: HomeSection + viewerUsername: string | undefined +} + +const JustAddedHomeSection = (props: HomeSectionProps): JSX.Element => { + const router = useRouter() + return ( + <VStack + distribution="start" + css={{ + width: '100%', + height: '100%', + gap: '20px', + }} + > + <HStack + css={{ + width: '100%', + lineHeight: '1', + '@mdDown': { + px: '20px', + }, + }} + distribution="start" + alignment="start" + > + <SpanBox + css={{ + fontFamily: '$inter', + fontSize: '16px', + fontWeight: '600', + color: '$homeTextTitle', + overflow: 'hidden', + textOverflow: 'ellipsis', + wordBreak: 'break-word', + display: '-webkit-box', + '-webkit-line-clamp': '2', + '-webkit-box-orient': 'vertical', + }} + > + {props.homeSection.title} + </SpanBox> + <SpanBox + css={{ + ml: 'auto', + fontFamily: '$inter', + fontSize: '13px', + fontWeight: '400', + color: '$homeTextTitle', + }} + > + <Button + style="link" + onClick={(event) => { + router.push('/l/library') + event.preventDefault() + }} + css={{ + '&:hover': { + textDecoration: 'underline', + }, + }} + > + View All + </Button> + </SpanBox> + </HStack> + <HStack + css={{ + width: '100%', + height: '100%', + lineHeight: '1', + overflowX: 'scroll', + gap: '25px', + scrollbarWidth: 'none', + '::-webkit-scrollbar': { + display: 'none', + }, + '@mdDown': { + px: '20px', + }, + }} + distribution="start" + alignment="start" + > + {props.homeSection.items.map((homeItem) => { + return <JustAddedItemView key={homeItem.id} homeItem={homeItem} /> + })} + </HStack> + </VStack> + ) +} + +const TopPicksHomeSection = (props: HomeSectionProps): JSX.Element => { + const listReducer = ( + state: HomeItem[], + action: { + type: string + itemId?: string + items?: HomeItem[] + } + ) => { + console.log('handling action: ', action) + switch (action.type) { + case 'RESET': + return action.items ?? [] + case 'REMOVE_ITEM': + return state.filter((item) => item.id !== action.itemId) + default: + throw new Error() + } + } + + const [items, dispatchList] = useReducer(listReducer, []) + + function handleDelete(item: HomeItem) { + dispatchList({ + type: 'REMOVE_ITEM', + itemId: item.id, + }) + } + + useEffect(() => { + dispatchList({ + type: 'RESET', + items: props.homeSection.items, + }) + }, [props]) + + return ( + <VStack + distribution="start" + css={{ + width: '100%', + gap: '20px', + '@mdDown': { + gap: '10px', + }, + }} + > + {items.length > 0 && ( + <SpanBox + css={{ + fontFamily: '$inter', + fontSize: '16px', + fontWeight: '600', + color: '$homeTextTitle', + '@mdDown': { + px: '20px', + }, + }} + > + {props.homeSection.title} + </SpanBox> + )} + + <Pagination + items={items} + itemsPerPage={10} + loadMoreButtonText="Load more Top Picks" + render={(homeItem) => ( + <TopPicksItemView + key={homeItem.id} + homeItem={homeItem} + dispatchList={dispatchList} + /> + )} + /> + </VStack> + ) +} + +const QuickLinksHomeSection = (props: HomeSectionProps): JSX.Element => { + return ( + <VStack + distribution="start" + css={{ + width: '100%', + gap: '10px', + bg: '$homeCardHover', + py: '20px', + px: '20px', + borderRadius: '5px', + }} + > + <SpanBox + css={{ + fontFamily: '$inter', + fontSize: '12px', + fontWeight: '500', + textTransform: 'uppercase', + color: '$ctaBlue', + bg: '#007AFF20', + px: '10px', + py: '5px', + borderRadius: '5px', + }} + > + {props.homeSection.title} + </SpanBox> + + <Pagination + items={props.homeSection.items} + itemsPerPage={8} + render={(homeItem) => ( + <QuickLinkHomeItemView key={homeItem.id} homeItem={homeItem} /> + )} + /> + </VStack> + ) +} + +const HiddenHomeSection = (props: HomeSectionProps): JSX.Element => { + const [isHidden, setIsHidden] = useState(true) + return ( + <VStack + distribution="start" + css={{ + width: '100%', + gap: '20px', + marginBottom: '40px', + }} + > + <HStack + distribution="start" + alignment="center" + css={{ + gap: '10px', + cursor: 'pointer', + }} + onClick={() => setIsHidden(!isHidden)} + > + <SpanBox + css={{ + fontFamily: '$inter', + fontSize: '16px', + fontWeight: '600', + color: '$homeTextTitle', + }} + > + {props.homeSection.title} + </SpanBox> + <SpanBox + css={{ + fontFamily: '$inter', + fontSize: '13px', + color: '$readerFont', + }} + > + {isHidden ? 'Show' : 'Hide'} + </SpanBox> + </HStack> + + {isHidden ? <></> : <HiddenHomeSectionView />} + </VStack> + ) +} + +const HiddenHomeSectionView = (): JSX.Element => { + const hiddenSectionData = useGetHiddenHomeSection() + + if (hiddenSectionData.error) { + return <SpanBox>Error loading hidden section</SpanBox> + } + + if (hiddenSectionData.isValidating) { + return <SpanBox>Loading...</SpanBox> + } + + if (!hiddenSectionData.section) { + return <SpanBox>No hidden section data</SpanBox> + } + + return ( + <VStack + distribution="start" + css={{ + width: '100%', + }} + > + {hiddenSectionData.section.items.map((homeItem) => { + return <QuickLinkHomeItemView key={homeItem.id} homeItem={homeItem} /> + })} + </VStack> + ) +} + +const CoverImage = styled('img', { + objectFit: 'cover', +}) + +type HomeItemViewProps = { + homeItem: HomeItem + viewerUsername?: string | undefined +} + +const TimeAgo = (props: HomeItemViewProps): JSX.Element => { + return ( + <HStack + distribution="start" + alignment="center" + css={{ + fontSize: '12px', + fontWeight: 'medium', + fontFamily: '$inter', + color: '$homeTextSubtle', + flexShrink: '0', + }} + > + {timeAgo(props.homeItem.date)} + </HStack> + ) +} + +const Title = (props: HomeItemViewProps): JSX.Element => { + return ( + <HStack + className="title-text" + distribution="start" + alignment="center" + css={{ + mb: '6px', + fontSize: '18px', + lineHeight: '24px', + fontWeight: '600', + fontFamily: '$inter', + color: '$homeTextTitle', + overflow: 'hidden', + textOverflow: 'ellipsis', + wordBreak: 'break-word', + display: '-webkit-box', + '-webkit-line-clamp': '3', + '-webkit-box-orient': 'vertical', + '&:title-text': { + transition: 'text-decoration 0.3s ease', + }, + '@mdDown': { + fontSize: '16px', + lineHeight: '20px', + }, + }} + > + {props.homeItem.title} + </HStack> + ) +} + +const TitleSmall = (props: HomeItemViewProps): JSX.Element => { + return ( + <HStack + className="title-text" + distribution="start" + alignment="center" + css={{ + fontSize: '14px', + lineHeight: '21px', + minHeight: '42px', // always have two lines of space + fontWeight: '500', + fontFamily: '$inter', + color: '$homeTextTitle', + overflow: 'hidden', + textOverflow: 'ellipsis', + wordBreak: 'break-word', + display: '-webkit-box', + '-webkit-line-clamp': '3', + '-webkit-box-orient': 'vertical', + }} + > + {props.homeItem.title} + </HStack> + ) +} + +type PreviewContentProps = { + previewContent?: string + maxLines?: string +} + +const PreviewContent = (props: PreviewContentProps): JSX.Element => { + return ( + <SpanBox + css={{ + fontFamily: '$inter', + fontSize: '14px', + lineHeight: '21px', + overflow: 'hidden', + textOverflow: 'ellipsis', + wordBreak: 'break-word', + display: '-webkit-box', + '-webkit-line-clamp': props.maxLines ?? '3', + '-webkit-box-orient': 'vertical', + '@mdDown': { + '-webkit-line-clamp': '3', + }, + }} + > + {props.previewContent ?? ''} + </SpanBox> + ) +} + +const JustAddedItemView = (props: HomeItemViewProps): JSX.Element => { + const router = useRouter() + + return ( + <VStack + css={{ + minWidth: '377px', + gap: '5px', + padding: '12px', + cursor: 'pointer', + bg: '$homeCardHover', + borderRadius: '5px', + '&:hover': { + bg: '$homeCardHover', + }, + '&:hover .title-text': { + textDecoration: 'underline', + }, + '@mdDown': { + minWidth: '282px', + }, + }} + onClick={(event) => { + const path = `/${props.viewerUsername ?? 'me'}/${props.homeItem.slug}` + if (event.metaKey || event.ctrlKey) { + window.open(path, '_blank') + } else { + router.push(path) + } + }} + > + <HStack + distribution="start" + alignment="center" + css={{ width: '100%', gap: '5px', lineHeight: '1' }} + > + <SourceInfo homeItem={props.homeItem} subtle={true} /> + <SpanBox css={{ ml: 'auto', flexShrink: '0' }}> + <TimeAgo homeItem={props.homeItem} /> + </SpanBox> + </HStack> + + <TitleSmall homeItem={props.homeItem} /> + </VStack> + ) +} + +type TopPicksItemViewProps = { + dispatchList: (args: { type: string; itemId?: string }) => void +} + +const TopPicksItemView = ( + props: HomeItemViewProps & TopPicksItemViewProps +): JSX.Element => { + const router = useRouter() + const { archiveItem, deleteItem, moveItem } = useLibraryItemActions() + + return ( + <VStack + css={{ + width: '100%', + pt: '15px', + cursor: 'pointer', + borderRadius: '5px', + '@mdDown': { + borderRadius: '0px', + }, + '&:hover': { + bg: '$homeCardHover', + }, + '&:hover .title-text': { + textDecoration: 'underline', + }, + }} + onClick={(event) => { + const path = `/${props.viewerUsername ?? 'me'}/${props.homeItem.slug}` + if (event.metaKey || event.ctrlKey) { + window.open(path, '_blank') + } else { + router.push(path) + } + }} + alignment="start" + > + <Box css={{ width: '100%', gap: '10px', px: '20px' }}> + <HStack + distribution="start" + alignment="center" + css={{ gap: '5px', lineHeight: '1', mb: '10px' }} + > + <SourceInfo homeItem={props.homeItem} /> + <SpanBox css={{ '@mdDown': { ml: 'auto' } }}> + <TimeAgo homeItem={props.homeItem} /> + </SpanBox> + </HStack> + + {props.homeItem.thumbnail && ( + <CoverImage + css={{ + width: '120px', + height: '70px', + borderRadius: '4px', + marginLeft: '10px', + float: 'right', + }} + src={props.homeItem.thumbnail} + ></CoverImage> + )} + <Title homeItem={props.homeItem} /> + <PreviewContent + previewContent={props.homeItem.previewContent} + maxLines="6" + /> + </Box> + <SpanBox css={{ px: '20px' }}></SpanBox> + <HStack css={{ gap: '10px', my: '15px', px: '20px' }}> + {props.homeItem.canSave && ( + <Button + style="homeAction" + onClick={async (event) => { + event.preventDefault() + event.stopPropagation() + + props.dispatchList({ + type: 'REMOVE_ITEM', + itemId: props.homeItem.id, + }) + if (!(await moveItem(props.homeItem.id))) { + props.dispatchList({ + type: 'REPLACE_ITEM', + itemId: props.homeItem.id, + }) + } + }} + > + <AddToLibraryActionIcon + color={theme.colors.homeActionIcons.toString()} + /> + </Button> + )} + {props.homeItem.canArchive && ( + <Button + style="homeAction" + onClick={async (event) => { + event.preventDefault() + event.stopPropagation() + + props.dispatchList({ + type: 'REMOVE_ITEM', + itemId: props.homeItem.id, + }) + if (!(await archiveItem(props.homeItem.id))) { + props.dispatchList({ + type: 'REPLACE_ITEM', + itemId: props.homeItem.id, + }) + } + }} + > + <ArchiveActionIcon + color={theme.colors.homeActionIcons.toString()} + /> + </Button> + )} + {props.homeItem.canDelete && ( + <Button + style="homeAction" + onClick={async (event) => { + event.preventDefault() + event.stopPropagation() + + props.dispatchList({ + type: 'REMOVE_ITEM', + itemId: props.homeItem.id, + }) + const undo = () => { + props.dispatchList({ + type: 'REPLACE_ITEM', + itemId: props.homeItem.id, + }) + } + if (!(await deleteItem(props.homeItem.id, undo))) { + props.dispatchList({ + type: 'REPLACE_ITEM', + itemId: props.homeItem.id, + }) + } + }} + > + <RemoveActionIcon color={theme.colors.homeActionIcons.toString()} /> + </Button> + )} + {props.homeItem.canShare && ( + <Button + style="homeAction" + onClick={(event) => { + event.preventDefault() + event.stopPropagation() + }} + > + <ShareActionIcon color={theme.colors.homeActionIcons.toString()} /> + </Button> + )} + </HStack> + <Box + css={{ mt: '15px', width: '100%', height: '1px', bg: '$homeDivider' }} + /> + </VStack> + ) +} + +const QuickLinkHomeItemView = (props: HomeItemViewProps): JSX.Element => { + const router = useRouter() + + return ( + <VStack + css={{ + mt: '10px', + width: '100%', + px: '10px', + py: '10px', + gap: '5px', + borderRadius: '5px', + '&:hover': { + bg: '#007AFF10', + cursor: 'pointer', + }, + '&:hover .title-text': { + textDecoration: 'underline', + }, + }} + onClick={(event) => { + const path = `/${props.viewerUsername ?? 'me'}/${props.homeItem.slug}` + if (event.metaKey || event.ctrlKey) { + window.open(path, '_blank') + } else { + router.push(path) + } + }} + > + <HStack + distribution="start" + alignment="center" + css={{ width: '100%', gap: '5px', lineHeight: '1' }} + > + <SourceInfo homeItem={props.homeItem} subtle={true} /> + + <SpanBox css={{ ml: 'auto', flexShrink: '0' }}> + <TimeAgo homeItem={props.homeItem} /> + </SpanBox> + </HStack> + <Title homeItem={props.homeItem} /> + <PreviewContent + previewContent={props.homeItem.previewContent} + maxLines="2" + /> + </VStack> + ) +} + +const SiteIconSmall = styled('img', { + width: '16px', + height: '16px', + borderRadius: '100px', +}) + +const SiteIconLarge = styled('img', { + width: '25px', + height: '25px', + borderRadius: '100px', +}) + +type SourceInfoProps = { + subtle?: boolean +} + +const SourceInfo = (props: HomeItemViewProps & SourceInfoProps) => ( + <HoverCard.Root> + <HoverCard.Trigger asChild> + <HStack + distribution="start" + alignment="center" + css={{ + gap: '8px', + height: '16px', + cursor: 'pointer', + flex: '1', + overflow: 'hidden', + whiteSpace: 'nowrap', + textOverflow: 'ellipsis', + }} + > + {props.homeItem.source.icon && ( + <SiteIconSmall src={props.homeItem.source.icon} /> + )} + <HStack + css={{ + lineHeight: '1', + fontFamily: '$inter', + fontWeight: '500', + fontSize: props.subtle ? '12px' : '13px', + color: props.subtle ? '$homeTextSubtle' : '$homeTextSource', + textDecoration: 'underline', + }} + > + {props.homeItem.source.name} + </HStack> + </HStack> + </HoverCard.Trigger> + <HoverCard.Portal> + <HoverCard.Content sideOffset={5} style={{ zIndex: 5 }}> + <SubscriptionSourceHoverContent source={props.homeItem.source} /> + <HoverCard.Arrow fill={theme.colors.thBackground2.toString()} /> + </HoverCard.Content> + </HoverCard.Portal> + </HoverCard.Root> +) + +type SourceHoverContentProps = { + source: HomeItemSource +} + +const SubscriptionSourceHoverContent = ( + props: SourceHoverContentProps +): JSX.Element => { + const mapSourceType = ( + sourceType: HomeItemSourceType + ): SubscriptionType | undefined => { + switch (sourceType) { + case 'RSS': + case 'NEWSLETTER': + return sourceType as SubscriptionType + default: + return undefined + } + } + const { subscriptions, isValidating } = useGetSubscriptionsQuery( + mapSourceType(props.source.type) + ) + const subscription = useMemo(() => { + if (props.source.id && subscriptions) { + return subscriptions.find((sub) => sub.id == props.source.id) + } + return undefined + }, [subscriptions]) + + return ( + <VStack + alignment="start" + distribution="start" + css={{ + width: '380px', + height: '200px', + bg: '$thBackground2', + borderRadius: '10px', + padding: '15px', + gap: '10px', + boxShadow: theme.shadows.cardBoxShadow.toString(), + }} + > + <HStack + distribution="start" + alignment="center" + css={{ width: '100%', gap: '10px', height: '35px' }} + > + {props.source.icon && <SiteIconLarge src={props.source.icon} />} + <SpanBox + css={{ + fontFamily: '$inter', + fontWeight: '500', + fontSize: '14px', + }} + > + {props.source.name} + </SpanBox> + <SpanBox css={{ ml: 'auto', minWidth: '100px' }}> + {subscription && subscription.status == 'ACTIVE' && ( + <Button style="ctaSubtle" css={{ fontSize: '12px' }}> + Unsubscribe + </Button> + )} + </SpanBox> + </HStack> + <SpanBox + css={{ + fontFamily: '$inter', + fontSize: '13px', + color: '$homeTextBody', + }} + > + {subscription ? <>{subscription.description}</> : <></>} + </SpanBox> + </VStack> + ) +} diff --git a/packages/web/components/tokens/stitches.config.ts b/packages/web/components/tokens/stitches.config.ts index 4351694c9..99dd8d755 100644 --- a/packages/web/components/tokens/stitches.config.ts +++ b/packages/web/components/tokens/stitches.config.ts @@ -416,7 +416,7 @@ const blackThemeSpec = { const apolloThemeSpec = { colors: { - readerBg: '#6A6968', + readerBg: '#474747', readerFont: '#F3F3F3', readerMargin: '#474747', readerFontHighContrast: 'white', diff --git a/packages/web/lib/hooks/useLibraryItemActions.tsx b/packages/web/lib/hooks/useLibraryItemActions.tsx new file mode 100644 index 000000000..d2b4d28ab --- /dev/null +++ b/packages/web/lib/hooks/useLibraryItemActions.tsx @@ -0,0 +1,69 @@ +import { useState, useEffect, useCallback } from 'react' +import { setLinkArchivedMutation } from '../networking/mutations/setLinkArchivedMutation' +import { + showErrorToast, + showSuccessToast, + showSuccessToastWithUndo, +} from '../toastHelpers' +import { deleteLinkMutation } from '../networking/mutations/deleteLinkMutation' +import { updatePageMutation } from '../networking/mutations/updatePageMutation' +import { State } from '../networking/fragments/articleFragment' + +export default function useLibraryItemActions() { + const archiveItem = useCallback(async (itemId: string) => { + const result = await setLinkArchivedMutation({ + linkId: itemId, + archived: true, + }) + + if (result) { + showSuccessToast('Link archived', { position: 'bottom-right' }) + } else { + showErrorToast('Error archiving link', { position: 'bottom-right' }) + } + + return !!result + }, []) + + const deleteItem = useCallback(async (itemId: string, undo: () => void) => { + const result = await deleteLinkMutation(itemId) + + if (result) { + showSuccessToastWithUndo('Item removed', async () => { + const result = await updatePageMutation({ + pageId: itemId, + state: State.SUCCEEDED, + }) + + undo() + + if (result) { + showSuccessToast('Item recovered') + } else { + showErrorToast('Error recovering, check your deleted items') + } + }) + } else { + showErrorToast('Error removing item', { position: 'bottom-right' }) + } + + return !!result + }, []) + + const moveItem = useCallback(async (itemId: string) => { + const result = await setLinkArchivedMutation({ + linkId: itemId, + archived: true, + }) + + if (result) { + showSuccessToast('Link archived', { position: 'bottom-right' }) + } else { + showErrorToast('Error archiving link', { position: 'bottom-right' }) + } + + return !!result + }, []) + + return { archiveItem, deleteItem, moveItem } +} diff --git a/packages/web/lib/networking/mutations/moveToLibraryMutation.ts b/packages/web/lib/networking/mutations/moveToLibraryMutation.ts new file mode 100644 index 000000000..205312dd6 --- /dev/null +++ b/packages/web/lib/networking/mutations/moveToLibraryMutation.ts @@ -0,0 +1,41 @@ +import { gql } from 'graphql-request' +import { gqlFetcher } from '../networkHelpers' + +type MoveToFolderResponseData = { + success?: boolean + errorCodes?: string[] +} + +type MoveToFolderResponse = { + moveToFolder?: MoveToFolderResponseData +} + +export async function moveToFolderMutation( + itemId: string, + folder: string +): Promise<boolean> { + const mutation = gql` + mutation MoveToFolder($id: ID!, $folder: String!) { + moveToFolder(id: $id, folder: $folder) { + ... on MoveToFolderSuccess { + success + } + ... on MoveToFolderError { + errorCodes + } + } + } + ` + + try { + const response = await gqlFetcher(mutation, { id: itemId, folder }) + const data = response as MoveToFolderResponse | undefined + if (data?.moveToFolder?.errorCodes) { + return false + } + return data?.moveToFolder?.success ?? false + } catch (error) { + console.log('MoveToFolder error', error) + return false + } +} From b0d6876e569d9297b7ba6598454e9e1b44f87185 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 13:31:50 +0800 Subject: [PATCH 18/32] add replication config --- packages/api/package.json | 2 +- packages/api/src/data_source.ts | 22 +++++ packages/api/src/util.ts | 16 ++++ yarn.lock | 145 +++++++++++++++++++++----------- 4 files changed, 137 insertions(+), 48 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index d5fc669a8..cfdd5dbcc 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -105,7 +105,7 @@ "snake-case": "^4.0.0", "supertest": "^6.2.2", "ts-loader": "^9.3.0", - "typeorm": "^0.3.4", + "typeorm": "^0.3.20", "typeorm-naming-strategies": "^4.1.0", "underscore": "^1.13.6", "url-pattern": "^1.0.3", diff --git a/packages/api/src/data_source.ts b/packages/api/src/data_source.ts index eeb57fd9c..c5c8e92bf 100644 --- a/packages/api/src/data_source.ts +++ b/packages/api/src/data_source.ts @@ -23,4 +23,26 @@ export const appDataSource = new DataSource({ max: env.pg.pool.max, idleTimeoutMillis: 10000, // 10 seconds }, + replication: env.pg.slave + ? { + // set the default destination for read queries as the master instance + defaultMode: 'master', + master: { + host: env.pg.host, + port: env.pg.port, + username: env.pg.userName, + password: env.pg.password, + database: env.pg.dbName, + }, + slaves: [ + { + host: env.pg.slave.host, + port: env.pg.slave.port, + username: env.pg.slave.userName, + password: env.pg.slave.password, + database: env.pg.slave.dbName, + }, + ], + } + : undefined, }) diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 096acf0a0..1742f1771 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -19,6 +19,13 @@ export interface BackendEnv { pool: { max: number } + slave?: { + host: string + port: number + userName: string + password: string + dbName: string + } } server: { jwtSecret: string @@ -218,6 +225,15 @@ export function getEnv(): BackendEnv { pool: { max: parseInt(parse('PG_POOL_MAX'), 10), }, + slave: parse('PG_SLAVE_HOST') + ? { + host: parse('PG_SLAVE_HOST'), + port: parseInt(parse('PG_SLAVE_PORT'), 10), + userName: parse('PG_SLAVE_USER'), + password: parse('PG_SLAVE_PASSWORD'), + dbName: parse('PG_SLAVE_DB'), + } + : undefined, } const server = { jwtSecret: parse('JWT_SECRET'), diff --git a/yarn.lock b/yarn.lock index 549cbf289..c582c3196 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6571,10 +6571,10 @@ "@smithy/util-buffer-from" "^2.2.0" tslib "^2.6.2" -"@sqltools/formatter@^1.2.2": - version "1.2.3" - resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.3.tgz#1185726610acc37317ddab11c3c7f9066966bd20" - integrity sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg== +"@sqltools/formatter@^1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.5.tgz#3abc203c79b8c3e90fd6c156a0c62d5403520e12" + integrity sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw== "@stitches/react@^1.2.5": version "1.2.8" @@ -10013,10 +10013,10 @@ app-root-dir@^1.0.2: resolved "https://registry.yarnpkg.com/app-root-dir/-/app-root-dir-1.0.2.tgz#38187ec2dea7577fff033ffcb12172692ff6e118" integrity sha1-OBh+wt6nV3//Az/8sSFyaS/24Rg= -app-root-path@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.0.0.tgz#210b6f43873227e18a4b810a032283311555d5ad" - integrity sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw== +app-root-path@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.1.0.tgz#5971a2fc12ba170369a7a1ef018c71e6e47c2e86" + integrity sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA== apparatus@^0.0.10: version "0.0.10" @@ -13503,7 +13503,7 @@ date-fns@^1.27.2: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c" integrity sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw== -date-fns@^2.16.1, date-fns@^2.28.0: +date-fns@^2.16.1: version "2.28.0" resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.28.0.tgz#9570d656f5fc13143e50c975a3b6bbeb46cd08b2" integrity sha512-8d35hViGYx/QH0icHYCeLmsLmMUheMmTyV9Fcm6gvNwdw31yXXH+O85sOBJ+OLnLQMKZowvpKb6FgMIQjcpvQw== @@ -13525,6 +13525,11 @@ dayjs@1.x, dayjs@^1.10.4, dayjs@^1.11.7: resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0" integrity sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ== +dayjs@^1.11.9: + version "1.11.11" + resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.11.tgz#dfe0e9d54c5f8b68ccf8ca5f72ac603e7e5ed59e" + integrity sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg== + debounce@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/debounce/-/debounce-1.2.0.tgz#44a540abc0ea9943018dc0eaa95cce87f65cd131" @@ -14255,17 +14260,12 @@ dotenv@^10.0.0: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81" integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== -dotenv@^16.0.0: - version "16.0.0" - resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.0.tgz#c619001253be89ebb638d027b609c75c26e47411" - integrity sha512-qD9WU0MPM4SWLPJy/r2Be+2WgQj8plChsyrCNQzW/0WjvcJQiKQJ9mH3ZgB3fxbUUxgc/11ZJ0Fi5KiimWGz2Q== - dotenv@^16.0.1: version "16.0.1" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d" integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ== -dotenv@^16.3.1: +dotenv@^16.0.3, dotenv@^16.3.1: version "16.4.5" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f" integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg== @@ -16942,7 +16942,7 @@ glob@7.1.7: once "^1.3.0" path-is-absolute "^1.0.0" -glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0: +glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: version "7.2.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== @@ -16965,6 +16965,18 @@ glob@^10.2.2: minipass "^5.0.0 || ^6.0.2 || ^7.0.0" path-scurry "^1.10.1" +glob@^10.3.10: + version "10.4.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.2.tgz#bed6b95dade5c1f80b4434daced233aee76160e5" + integrity sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w== + dependencies: + foreground-child "^3.1.0" + jackspeak "^3.1.2" + minimatch "^9.0.4" + minipass "^7.1.2" + package-json-from-dist "^1.0.0" + path-scurry "^1.11.1" + glob@^8.0.0: version "8.0.3" resolved "https://registry.yarnpkg.com/glob/-/glob-8.0.3.tgz#415c6eb2deed9e502c68fa44a272e6da6eeca42e" @@ -19443,6 +19455,15 @@ jackspeak@^2.3.5: optionalDependencies: "@pkgjs/parseargs" "^0.11.0" +jackspeak@^3.1.2: + version "3.4.0" + resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.4.0.tgz#a75763ff36ad778ede6a156d8ee8b124de445b4a" + integrity sha512-JVYhQnN59LVPFCEcVa2C3CrEKYacvjRfqIQl+h8oi91aLYQVWRYbxjPcv1bUiUy/kLmQaANrYfNMCO3kuEDHfw== + dependencies: + "@isaacs/cliui" "^8.0.2" + optionalDependencies: + "@pkgjs/parseargs" "^0.11.0" + jaeger-client@^3.15.0: version "3.18.1" resolved "https://registry.yarnpkg.com/jaeger-client/-/jaeger-client-3.18.1.tgz#a8c7a778244ba117f4fb8775eb6aa5508703564e" @@ -21436,6 +21457,11 @@ lowlight@^1.14.0: fault "^1.0.0" highlight.js "~10.7.0" +lru-cache@^10.2.0: + version "10.2.2" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.2.2.tgz#48206bc114c1252940c41b25b41af5b545aca878" + integrity sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ== + lru-cache@^4.1.5: version "4.1.5" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd" @@ -22557,6 +22583,13 @@ minimatch@^9.0.0, minimatch@^9.0.1: dependencies: brace-expansion "^2.0.1" +minimatch@^9.0.4: + version "9.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.4.tgz#8e49c731d1749cbec05050ee5145147b32496a51" + integrity sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw== + dependencies: + brace-expansion "^2.0.1" + minimist-options@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619" @@ -22663,6 +22696,11 @@ minipass@^5.0.0: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== +minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + ministyle@~0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/ministyle/-/ministyle-0.1.4.tgz#b10481eb16aa8f7b6cd983817393a44da0e5a0cd" @@ -22746,6 +22784,11 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== +mkdirp@^2.1.3: + version "2.1.6" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-2.1.6.tgz#964fbcb12b2d8c5d6fbc62a963ac95a273e2cc19" + integrity sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A== + mkdirp@~0.3.5: version "0.3.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.5.tgz#de3e5f8961c88c787ee1368df849ac4413eca8d7" @@ -24703,6 +24746,11 @@ package-hash@^4.0.0: lodash.flattendeep "^4.4.0" release-zalgo "^1.0.0" +package-json-from-dist@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz#e501cd3094b278495eb4258d4c9f6d5ac3019f00" + integrity sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw== + package-json@^6.3.0: version "6.5.0" resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0" @@ -25074,6 +25122,14 @@ path-scurry@^1.10.1, path-scurry@^1.6.1: lru-cache "^9.1.1 || ^10.0.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" +path-scurry@^1.11.1: + version "1.11.1" + resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" + integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== + dependencies: + lru-cache "^10.2.0" + minipass "^5.0.0 || ^6.0.2 || ^7.0.0" + path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -27315,10 +27371,10 @@ redux@^5.0.0: resolved "https://registry.yarnpkg.com/redux/-/redux-5.0.1.tgz#97fa26881ce5746500125585d5642c77b6e9447b" integrity sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w== -reflect-metadata@^0.1.13: - version "0.1.13" - resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08" - integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg== +reflect-metadata@^0.2.1: + version "0.2.2" + resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.2.2.tgz#400c845b6cba87a21f2c65c4aeb158f4fa4d9c5b" + integrity sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q== reflect.getprototypeof@^1.0.4: version "1.0.4" @@ -30417,6 +30473,11 @@ tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tslib@^2.5.0: + version "2.6.3" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0" + integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ== + tslib@~2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c" @@ -30628,28 +30689,26 @@ typeorm-naming-strategies@^4.1.0: resolved "https://registry.yarnpkg.com/typeorm-naming-strategies/-/typeorm-naming-strategies-4.1.0.tgz#1ec6eb296c8d7b69bb06764d5b9083ff80e814a9" integrity sha512-vPekJXzZOTZrdDvTl1YoM+w+sUIfQHG4kZTpbFYoTsufyv9NIBRe4Q+PdzhEAFA2std3D9LZHEb1EjE9zhRpiQ== -typeorm@^0.3.4: - version "0.3.7" - resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.7.tgz#5776ed5058f0acb75d64723b39ff458d21de64c1" - integrity sha512-MsPJeP6Zuwfe64c++l80+VRqpGEGxf0CkztIEnehQ+CMmQPSHjOnFbFxwBuZ2jiLqZTjLk2ZqQdVF0RmvxNF3Q== +typeorm@^0.3.20: + version "0.3.20" + resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.20.tgz#4b61d737c6fed4e9f63006f88d58a5e54816b7ab" + integrity sha512-sJ0T08dV5eoZroaq9uPKBoNcGslHBR4E4y+EBHs//SiGbblGe7IeduP/IH4ddCcj0qp3PHwDwGnuvqEAnKlq/Q== dependencies: - "@sqltools/formatter" "^1.2.2" - app-root-path "^3.0.0" + "@sqltools/formatter" "^1.2.5" + app-root-path "^3.1.0" buffer "^6.0.3" - chalk "^4.1.0" + chalk "^4.1.2" cli-highlight "^2.1.11" - date-fns "^2.28.0" - debug "^4.3.3" - dotenv "^16.0.0" - glob "^7.2.0" - js-yaml "^4.1.0" - mkdirp "^1.0.4" - reflect-metadata "^0.1.13" + dayjs "^1.11.9" + debug "^4.3.4" + dotenv "^16.0.3" + glob "^10.3.10" + mkdirp "^2.1.3" + reflect-metadata "^0.2.1" sha.js "^2.4.11" - tslib "^2.3.1" - uuid "^8.3.2" - xml2js "^0.4.23" - yargs "^17.3.1" + tslib "^2.5.0" + uuid "^9.0.0" + yargs "^17.6.2" typescript@4.5.2: version "4.5.2" @@ -32212,14 +32271,6 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== -xml2js@^0.4.23: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - xml2js@^0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" @@ -32392,7 +32443,7 @@ yargs@^15.0.2, yargs@^15.3.1: y18n "^4.0.0" yargs-parser "^18.1.2" -yargs@^17.0.0, yargs@^17.3.1: +yargs@^17.0.0: version "17.4.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.4.0.tgz#9fc9efc96bd3aa2c1240446af28499f0e7593d00" integrity sha512-WJudfrk81yWFSOkZYpAZx4Nt7V4xp7S/uJkX0CnxovMCt1wCE8LNftPpNuF9X/u9gN5nsD7ycYtRcDf2pL3UiA== From a1cd611fa75a5604eb7d8913b1974dd6731ba1c8 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 14:39:40 +0800 Subject: [PATCH 19/32] use read replica for "read" --- packages/api/src/data_source.ts | 45 +++++++++++++++++---------------- packages/api/src/util.ts | 26 +++++++++++-------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/packages/api/src/data_source.ts b/packages/api/src/data_source.ts index c5c8e92bf..d18eda8f8 100644 --- a/packages/api/src/data_source.ts +++ b/packages/api/src/data_source.ts @@ -23,26 +23,27 @@ export const appDataSource = new DataSource({ max: env.pg.pool.max, idleTimeoutMillis: 10000, // 10 seconds }, - replication: env.pg.slave - ? { - // set the default destination for read queries as the master instance - defaultMode: 'master', - master: { - host: env.pg.host, - port: env.pg.port, - username: env.pg.userName, - password: env.pg.password, - database: env.pg.dbName, - }, - slaves: [ - { - host: env.pg.slave.host, - port: env.pg.slave.port, - username: env.pg.slave.userName, - password: env.pg.slave.password, - database: env.pg.slave.dbName, - }, - ], - } - : undefined, }) + +if (env.pg.replication) { + appDataSource.setOptions({ + replication: { + master: { + host: env.pg.host, + port: env.pg.port, + username: env.pg.userName, + password: env.pg.password, + database: env.pg.dbName, + }, + slaves: [ + { + host: env.pg.slave.host, + port: env.pg.slave.port, + username: env.pg.slave.userName, + password: env.pg.slave.password, + database: env.pg.slave.dbName, + }, + ], + }, + }) +} diff --git a/packages/api/src/util.ts b/packages/api/src/util.ts index 1742f1771..4e32d5406 100755 --- a/packages/api/src/util.ts +++ b/packages/api/src/util.ts @@ -19,7 +19,8 @@ export interface BackendEnv { pool: { max: number } - slave?: { + replication: boolean + slave: { host: string port: number userName: string @@ -186,6 +187,12 @@ const nullableEnvVars = [ 'NOTION_CLIENT_SECRET', 'NOTION_AUTH_URL', 'SCORE_API_URL', + 'PG_REPLICATION', + 'PG_SLAVE_HOST', + 'PG_SLAVE_PORT', + 'PG_SLAVE_USER', + 'PG_SLAVE_PASSWORD', + 'PG_SLAVE_DB', ] // Allow some vars to be null/empty const envParser = @@ -225,15 +232,14 @@ export function getEnv(): BackendEnv { pool: { max: parseInt(parse('PG_POOL_MAX'), 10), }, - slave: parse('PG_SLAVE_HOST') - ? { - host: parse('PG_SLAVE_HOST'), - port: parseInt(parse('PG_SLAVE_PORT'), 10), - userName: parse('PG_SLAVE_USER'), - password: parse('PG_SLAVE_PASSWORD'), - dbName: parse('PG_SLAVE_DB'), - } - : undefined, + replication: parse('PG_REPLICATION') === 'true', + slave: { + host: parse('PG_SLAVE_HOST'), + port: parseInt(parse('PG_SLAVE_PORT'), 10), + userName: parse('PG_SLAVE_USER'), + password: parse('PG_SLAVE_PASSWORD'), + dbName: parse('PG_SLAVE_DB'), + }, } const server = { jwtSecret: parse('JWT_SECRET'), From bc934a4706ca2f80ca3e1fb271bb264fedf630b3 Mon Sep 17 00:00:00 2001 From: Jackson Harper <jacksonh@gmail.com> Date: Fri, 21 Jun 2024 15:07:21 +0800 Subject: [PATCH 20/32] Make sure recently added has a max of two lines --- .../web/components/nav-containers/HomeContainer.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/web/components/nav-containers/HomeContainer.tsx b/packages/web/components/nav-containers/HomeContainer.tsx index 59ff4e84a..0c1da5be9 100644 --- a/packages/web/components/nav-containers/HomeContainer.tsx +++ b/packages/web/components/nav-containers/HomeContainer.tsx @@ -469,7 +469,13 @@ const Title = (props: HomeItemViewProps): JSX.Element => { ) } -const TitleSmall = (props: HomeItemViewProps): JSX.Element => { +type TitleSmallProps = { + maxLines?: string +} + +const TitleSmall = ( + props: HomeItemViewProps & TitleSmallProps +): JSX.Element => { return ( <HStack className="title-text" @@ -486,7 +492,7 @@ const TitleSmall = (props: HomeItemViewProps): JSX.Element => { textOverflow: 'ellipsis', wordBreak: 'break-word', display: '-webkit-box', - '-webkit-line-clamp': '3', + '-webkit-line-clamp': props.maxLines ?? '3', '-webkit-box-orient': 'vertical', }} > @@ -565,7 +571,7 @@ const JustAddedItemView = (props: HomeItemViewProps): JSX.Element => { </SpanBox> </HStack> - <TitleSmall homeItem={props.homeItem} /> + <TitleSmall homeItem={props.homeItem} maxLines="2" /> </VStack> ) } From b39bffdc1a2dc2bbecc88f177263be495adc2a02 Mon Sep 17 00:00:00 2001 From: Jackson Harper <jacksonh@gmail.com> Date: Fri, 21 Jun 2024 15:31:57 +0800 Subject: [PATCH 21/32] Add hover bg for apollo, implement share --- .../nav-containers/HomeContainer.tsx | 9 ++++--- .../web/components/tokens/stitches.config.ts | 2 +- .../web/lib/hooks/useLibraryItemActions.tsx | 24 ++++++++++++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/web/components/nav-containers/HomeContainer.tsx b/packages/web/components/nav-containers/HomeContainer.tsx index 0c1da5be9..2dbd80152 100644 --- a/packages/web/components/nav-containers/HomeContainer.tsx +++ b/packages/web/components/nav-containers/HomeContainer.tsx @@ -542,7 +542,7 @@ const JustAddedItemView = (props: HomeItemViewProps): JSX.Element => { bg: '$homeCardHover', borderRadius: '5px', '&:hover': { - bg: '$homeCardHover', + bg: '#007AFF10', }, '&:hover .title-text': { textDecoration: 'underline', @@ -584,7 +584,8 @@ const TopPicksItemView = ( props: HomeItemViewProps & TopPicksItemViewProps ): JSX.Element => { const router = useRouter() - const { archiveItem, deleteItem, moveItem } = useLibraryItemActions() + const { archiveItem, deleteItem, moveItem, shareItem } = + useLibraryItemActions() return ( <VStack @@ -724,9 +725,11 @@ const TopPicksItemView = ( {props.homeItem.canShare && ( <Button style="homeAction" - onClick={(event) => { + onClick={async (event) => { event.preventDefault() event.stopPropagation() + + await shareItem(props.homeItem.title, props.homeItem.url) }} > <ShareActionIcon color={theme.colors.homeActionIcons.toString()} /> diff --git a/packages/web/components/tokens/stitches.config.ts b/packages/web/components/tokens/stitches.config.ts index 99dd8d755..d07b70932 100644 --- a/packages/web/components/tokens/stitches.config.ts +++ b/packages/web/components/tokens/stitches.config.ts @@ -431,7 +431,7 @@ const apolloThemeSpec = { homeCardHover: '#525252', homeDivider: '#6A6968', - homeActionHoverBg: '#515151', + homeActionHoverBg: '#474747', thBackground: '#474747', thBackground2: '#515151', diff --git a/packages/web/lib/hooks/useLibraryItemActions.tsx b/packages/web/lib/hooks/useLibraryItemActions.tsx index d2b4d28ab..8be217b34 100644 --- a/packages/web/lib/hooks/useLibraryItemActions.tsx +++ b/packages/web/lib/hooks/useLibraryItemActions.tsx @@ -65,5 +65,27 @@ export default function useLibraryItemActions() { return !!result }, []) - return { archiveItem, deleteItem, moveItem } + const shareItem = useCallback( + async (title: string, originalArticleUrl: string | undefined) => { + if (!originalArticleUrl) { + showErrorToast('Article has no public URL to share', { + position: 'bottom-right', + }) + } else if (navigator.share) { + navigator.share({ + title: title + '\n', + text: title + '\n', + url: originalArticleUrl, + }) + } else { + await navigator.clipboard.writeText(originalArticleUrl) + showSuccessToast('URL copied to clipboard', { + position: 'bottom-right', + }) + } + }, + [] + ) + + return { archiveItem, deleteItem, moveItem, shareItem } } From a3cf333730b1da6aeeaabfc12288097c767ba25b Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 17:25:20 +0800 Subject: [PATCH 22/32] revert change to typeorm --- packages/api/package.json | 2 +- yarn.lock | 145 ++++++++++++-------------------------- 2 files changed, 48 insertions(+), 99 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index cfdd5dbcc..d5fc669a8 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -105,7 +105,7 @@ "snake-case": "^4.0.0", "supertest": "^6.2.2", "ts-loader": "^9.3.0", - "typeorm": "^0.3.20", + "typeorm": "^0.3.4", "typeorm-naming-strategies": "^4.1.0", "underscore": "^1.13.6", "url-pattern": "^1.0.3", diff --git a/yarn.lock b/yarn.lock index c582c3196..549cbf289 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6571,10 +6571,10 @@ "@smithy/util-buffer-from" "^2.2.0" tslib "^2.6.2" -"@sqltools/formatter@^1.2.5": - version "1.2.5" - resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.5.tgz#3abc203c79b8c3e90fd6c156a0c62d5403520e12" - integrity sha512-Uy0+khmZqUrUGm5dmMqVlnvufZRSK0FbYzVgp0UMstm+F5+W2/jnEEQyc9vo1ZR/E5ZI/B1WjjoTqBqwJL6Krw== +"@sqltools/formatter@^1.2.2": + version "1.2.3" + resolved "https://registry.yarnpkg.com/@sqltools/formatter/-/formatter-1.2.3.tgz#1185726610acc37317ddab11c3c7f9066966bd20" + integrity sha512-O3uyB/JbkAEMZaP3YqyHH7TMnex7tWyCbCI4EfJdOCoN6HIhqdJBWTM6aCCiWQ/5f5wxjgU735QAIpJbjDvmzg== "@stitches/react@^1.2.5": version "1.2.8" @@ -10013,10 +10013,10 @@ app-root-dir@^1.0.2: resolved "https://registry.yarnpkg.com/app-root-dir/-/app-root-dir-1.0.2.tgz#38187ec2dea7577fff033ffcb12172692ff6e118" integrity sha1-OBh+wt6nV3//Az/8sSFyaS/24Rg= -app-root-path@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.1.0.tgz#5971a2fc12ba170369a7a1ef018c71e6e47c2e86" - integrity sha512-biN3PwB2gUtjaYy/isrU3aNWI5w+fAfvHkSvCKeQGxhmYpwKFUxudR3Yya+KqVRHBmEDYh+/lTozYCFbmzX4nA== +app-root-path@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-3.0.0.tgz#210b6f43873227e18a4b810a032283311555d5ad" + integrity sha512-qMcx+Gy2UZynHjOHOIXPNvpf+9cjvk3cWrBBK7zg4gH9+clobJRb9NGzcT7mQTcV/6Gm/1WelUtqxVXnNlrwcw== apparatus@^0.0.10: version "0.0.10" @@ -13503,7 +13503,7 @@ date-fns@^1.27.2: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c" integrity sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw== -date-fns@^2.16.1: +date-fns@^2.16.1, date-fns@^2.28.0: version "2.28.0" resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.28.0.tgz#9570d656f5fc13143e50c975a3b6bbeb46cd08b2" integrity sha512-8d35hViGYx/QH0icHYCeLmsLmMUheMmTyV9Fcm6gvNwdw31yXXH+O85sOBJ+OLnLQMKZowvpKb6FgMIQjcpvQw== @@ -13525,11 +13525,6 @@ dayjs@1.x, dayjs@^1.10.4, dayjs@^1.11.7: resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0" integrity sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ== -dayjs@^1.11.9: - version "1.11.11" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.11.tgz#dfe0e9d54c5f8b68ccf8ca5f72ac603e7e5ed59e" - integrity sha512-okzr3f11N6WuqYtZSvm+F776mB41wRZMhKP+hc34YdW+KmtYYK9iqvHSwo2k9FEH3fhGXvOPV6yz2IcSrfRUDg== - debounce@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/debounce/-/debounce-1.2.0.tgz#44a540abc0ea9943018dc0eaa95cce87f65cd131" @@ -14260,12 +14255,17 @@ dotenv@^10.0.0: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-10.0.0.tgz#3d4227b8fb95f81096cdd2b66653fb2c7085ba81" integrity sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q== +dotenv@^16.0.0: + version "16.0.0" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.0.tgz#c619001253be89ebb638d027b609c75c26e47411" + integrity sha512-qD9WU0MPM4SWLPJy/r2Be+2WgQj8plChsyrCNQzW/0WjvcJQiKQJ9mH3ZgB3fxbUUxgc/11ZJ0Fi5KiimWGz2Q== + dotenv@^16.0.1: version "16.0.1" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.0.1.tgz#8f8f9d94876c35dac989876a5d3a82a267fdce1d" integrity sha512-1K6hR6wtk2FviQ4kEiSjFiH5rpzEVi8WW0x96aztHVMhEspNpc4DVOUTEHtEva5VThQ8IaBX1Pe4gSzpVVUsKQ== -dotenv@^16.0.3, dotenv@^16.3.1: +dotenv@^16.3.1: version "16.4.5" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f" integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg== @@ -16942,7 +16942,7 @@ glob@7.1.7: once "^1.3.0" path-is-absolute "^1.0.0" -glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: +glob@7.2.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.2.0: version "7.2.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023" integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q== @@ -16965,18 +16965,6 @@ glob@^10.2.2: minipass "^5.0.0 || ^6.0.2 || ^7.0.0" path-scurry "^1.10.1" -glob@^10.3.10: - version "10.4.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.2.tgz#bed6b95dade5c1f80b4434daced233aee76160e5" - integrity sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w== - dependencies: - foreground-child "^3.1.0" - jackspeak "^3.1.2" - minimatch "^9.0.4" - minipass "^7.1.2" - package-json-from-dist "^1.0.0" - path-scurry "^1.11.1" - glob@^8.0.0: version "8.0.3" resolved "https://registry.yarnpkg.com/glob/-/glob-8.0.3.tgz#415c6eb2deed9e502c68fa44a272e6da6eeca42e" @@ -19455,15 +19443,6 @@ jackspeak@^2.3.5: optionalDependencies: "@pkgjs/parseargs" "^0.11.0" -jackspeak@^3.1.2: - version "3.4.0" - resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.4.0.tgz#a75763ff36ad778ede6a156d8ee8b124de445b4a" - integrity sha512-JVYhQnN59LVPFCEcVa2C3CrEKYacvjRfqIQl+h8oi91aLYQVWRYbxjPcv1bUiUy/kLmQaANrYfNMCO3kuEDHfw== - dependencies: - "@isaacs/cliui" "^8.0.2" - optionalDependencies: - "@pkgjs/parseargs" "^0.11.0" - jaeger-client@^3.15.0: version "3.18.1" resolved "https://registry.yarnpkg.com/jaeger-client/-/jaeger-client-3.18.1.tgz#a8c7a778244ba117f4fb8775eb6aa5508703564e" @@ -21457,11 +21436,6 @@ lowlight@^1.14.0: fault "^1.0.0" highlight.js "~10.7.0" -lru-cache@^10.2.0: - version "10.2.2" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.2.2.tgz#48206bc114c1252940c41b25b41af5b545aca878" - integrity sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ== - lru-cache@^4.1.5: version "4.1.5" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd" @@ -22583,13 +22557,6 @@ minimatch@^9.0.0, minimatch@^9.0.1: dependencies: brace-expansion "^2.0.1" -minimatch@^9.0.4: - version "9.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.4.tgz#8e49c731d1749cbec05050ee5145147b32496a51" - integrity sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw== - dependencies: - brace-expansion "^2.0.1" - minimist-options@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619" @@ -22696,11 +22663,6 @@ minipass@^5.0.0: resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.0.4.tgz#dbce03740f50a4786ba994c1fb908844d27b038c" integrity sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ== -minipass@^7.1.2: - version "7.1.2" - resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" - integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== - ministyle@~0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/ministyle/-/ministyle-0.1.4.tgz#b10481eb16aa8f7b6cd983817393a44da0e5a0cd" @@ -22784,11 +22746,6 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== -mkdirp@^2.1.3: - version "2.1.6" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-2.1.6.tgz#964fbcb12b2d8c5d6fbc62a963ac95a273e2cc19" - integrity sha512-+hEnITedc8LAtIP9u3HJDFIdcLV2vXP33sqLLIzkv1Db1zO/1OxbvYf0Y1OC/S/Qo5dxHXepofhmxL02PsKe+A== - mkdirp@~0.3.5: version "0.3.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.5.tgz#de3e5f8961c88c787ee1368df849ac4413eca8d7" @@ -24746,11 +24703,6 @@ package-hash@^4.0.0: lodash.flattendeep "^4.4.0" release-zalgo "^1.0.0" -package-json-from-dist@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz#e501cd3094b278495eb4258d4c9f6d5ac3019f00" - integrity sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw== - package-json@^6.3.0: version "6.5.0" resolved "https://registry.yarnpkg.com/package-json/-/package-json-6.5.0.tgz#6feedaca35e75725876d0b0e64974697fed145b0" @@ -25122,14 +25074,6 @@ path-scurry@^1.10.1, path-scurry@^1.6.1: lru-cache "^9.1.1 || ^10.0.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" -path-scurry@^1.11.1: - version "1.11.1" - resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" - integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== - dependencies: - lru-cache "^10.2.0" - minipass "^5.0.0 || ^6.0.2 || ^7.0.0" - path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" @@ -27371,10 +27315,10 @@ redux@^5.0.0: resolved "https://registry.yarnpkg.com/redux/-/redux-5.0.1.tgz#97fa26881ce5746500125585d5642c77b6e9447b" integrity sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w== -reflect-metadata@^0.2.1: - version "0.2.2" - resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.2.2.tgz#400c845b6cba87a21f2c65c4aeb158f4fa4d9c5b" - integrity sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q== +reflect-metadata@^0.1.13: + version "0.1.13" + resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08" + integrity sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg== reflect.getprototypeof@^1.0.4: version "1.0.4" @@ -30473,11 +30417,6 @@ tslib@^2, tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0, tslib@^2.3.0, resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== -tslib@^2.5.0: - version "2.6.3" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0" - integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ== - tslib@~2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c" @@ -30689,26 +30628,28 @@ typeorm-naming-strategies@^4.1.0: resolved "https://registry.yarnpkg.com/typeorm-naming-strategies/-/typeorm-naming-strategies-4.1.0.tgz#1ec6eb296c8d7b69bb06764d5b9083ff80e814a9" integrity sha512-vPekJXzZOTZrdDvTl1YoM+w+sUIfQHG4kZTpbFYoTsufyv9NIBRe4Q+PdzhEAFA2std3D9LZHEb1EjE9zhRpiQ== -typeorm@^0.3.20: - version "0.3.20" - resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.20.tgz#4b61d737c6fed4e9f63006f88d58a5e54816b7ab" - integrity sha512-sJ0T08dV5eoZroaq9uPKBoNcGslHBR4E4y+EBHs//SiGbblGe7IeduP/IH4ddCcj0qp3PHwDwGnuvqEAnKlq/Q== +typeorm@^0.3.4: + version "0.3.7" + resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.3.7.tgz#5776ed5058f0acb75d64723b39ff458d21de64c1" + integrity sha512-MsPJeP6Zuwfe64c++l80+VRqpGEGxf0CkztIEnehQ+CMmQPSHjOnFbFxwBuZ2jiLqZTjLk2ZqQdVF0RmvxNF3Q== dependencies: - "@sqltools/formatter" "^1.2.5" - app-root-path "^3.1.0" + "@sqltools/formatter" "^1.2.2" + app-root-path "^3.0.0" buffer "^6.0.3" - chalk "^4.1.2" + chalk "^4.1.0" cli-highlight "^2.1.11" - dayjs "^1.11.9" - debug "^4.3.4" - dotenv "^16.0.3" - glob "^10.3.10" - mkdirp "^2.1.3" - reflect-metadata "^0.2.1" + date-fns "^2.28.0" + debug "^4.3.3" + dotenv "^16.0.0" + glob "^7.2.0" + js-yaml "^4.1.0" + mkdirp "^1.0.4" + reflect-metadata "^0.1.13" sha.js "^2.4.11" - tslib "^2.5.0" - uuid "^9.0.0" - yargs "^17.6.2" + tslib "^2.3.1" + uuid "^8.3.2" + xml2js "^0.4.23" + yargs "^17.3.1" typescript@4.5.2: version "4.5.2" @@ -32271,6 +32212,14 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@^0.4.23: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + xml2js@^0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" @@ -32443,7 +32392,7 @@ yargs@^15.0.2, yargs@^15.3.1: y18n "^4.0.0" yargs-parser "^18.1.2" -yargs@^17.0.0: +yargs@^17.0.0, yargs@^17.3.1: version "17.4.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.4.0.tgz#9fc9efc96bd3aa2c1240446af28499f0e7593d00" integrity sha512-WJudfrk81yWFSOkZYpAZx4Nt7V4xp7S/uJkX0CnxovMCt1wCE8LNftPpNuF9X/u9gN5nsD7ycYtRcDf2pL3UiA== From 67fc0baaed632155d0115bfc9f907ebb87c29130 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 17:41:59 +0800 Subject: [PATCH 23/32] fix: lint migration github action failed for pull request from contributor --- .github/workflows/lint-migrations.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-migrations.yml b/.github/workflows/lint-migrations.yml index c1587f696..5770af018 100644 --- a/.github/workflows/lint-migrations.yml +++ b/.github/workflows/lint-migrations.yml @@ -10,9 +10,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v1 + - name: Fetch main branch + run: git fetch origin main:main - name: Find modified migrations run: | - modified_migrations=$(git diff --diff-filter=d --name-only origin/$GITHUB_BASE_REF...origin/$GITHUB_HEAD_REF 'packages/db/migrations/*.do.*.sql') + modified_migrations=$(git diff --diff-filter=d --name-only main 'packages/db/migrations/*.do.*.sql') echo "$modified_migrations" echo "::set-output name=file_names::$modified_migrations" id: modified-migrations From f64e85be5bc713d4664d6b746f249584c16f7bd3 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 17:44:19 +0800 Subject: [PATCH 24/32] create a test migration sql --- packages/db/migrations/0183.do.test.sql | 14 ++++++++++++++ packages/db/migrations/0183.undo.test.sql | 7 +++++++ 2 files changed, 21 insertions(+) create mode 100755 packages/db/migrations/0183.do.test.sql create mode 100755 packages/db/migrations/0183.undo.test.sql diff --git a/packages/db/migrations/0183.do.test.sql b/packages/db/migrations/0183.do.test.sql new file mode 100755 index 000000000..5ac85ac63 --- /dev/null +++ b/packages/db/migrations/0183.do.test.sql @@ -0,0 +1,14 @@ +-- Type: DO +-- Name: test +-- Description: test + +BEGIN; + +CREATE TABLE omnivore.test ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v1mc(), + name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +COMMIT; diff --git a/packages/db/migrations/0183.undo.test.sql b/packages/db/migrations/0183.undo.test.sql new file mode 100755 index 000000000..811a356c3 --- /dev/null +++ b/packages/db/migrations/0183.undo.test.sql @@ -0,0 +1,7 @@ +-- Type: UNDO +-- Name: test +-- Description: test + +BEGIN; + +COMMIT; From 38f7317cf8eb0cedf5c6c9da5fea58c183e1b950 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 17:50:12 +0800 Subject: [PATCH 25/32] replace set-output command with github env var --- .github/workflows/lint-migrations.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint-migrations.yml b/.github/workflows/lint-migrations.yml index 5770af018..24df2ad47 100644 --- a/.github/workflows/lint-migrations.yml +++ b/.github/workflows/lint-migrations.yml @@ -16,8 +16,7 @@ jobs: run: | modified_migrations=$(git diff --diff-filter=d --name-only main 'packages/db/migrations/*.do.*.sql') echo "$modified_migrations" - echo "::set-output name=file_names::$modified_migrations" - id: modified-migrations + echo "{FILE_NAMES}={$modified_migrations}" >> $GITHUB_OUTPUT - uses: sbdchd/squawk-action@v1 with: - pattern: ${{ steps.modified-migrations.outputs.file_names }} + pattern: $FILE_NAMES From 81da0eee4f60520eb55fa0dd6acfba8eca5b8f16 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 17:52:06 +0800 Subject: [PATCH 26/32] more testing --- packages/db/migrations/0183.do.test.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/db/migrations/0183.do.test.sql b/packages/db/migrations/0183.do.test.sql index 5ac85ac63..872c82fbe 100755 --- a/packages/db/migrations/0183.do.test.sql +++ b/packages/db/migrations/0183.do.test.sql @@ -11,4 +11,6 @@ CREATE TABLE omnivore.test ( updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP ); +CREATE TRIGGER update_test_modtime BEFORE UPDATE ON omnivore.test FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + COMMIT; From 96d6fb965366ca77dce65b2483d9cfb7ab6deb58 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 17:59:25 +0800 Subject: [PATCH 27/32] use github env var correctly --- .github/workflows/lint-migrations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-migrations.yml b/.github/workflows/lint-migrations.yml index 24df2ad47..3105ff9ed 100644 --- a/.github/workflows/lint-migrations.yml +++ b/.github/workflows/lint-migrations.yml @@ -19,4 +19,4 @@ jobs: echo "{FILE_NAMES}={$modified_migrations}" >> $GITHUB_OUTPUT - uses: sbdchd/squawk-action@v1 with: - pattern: $FILE_NAMES + pattern: "$FILE_NAMES" From a15c6793117f8fbe76d96bb8634c96864a535705 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 18:03:26 +0800 Subject: [PATCH 28/32] fix typo --- .github/workflows/lint-migrations.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-migrations.yml b/.github/workflows/lint-migrations.yml index 3105ff9ed..00350abcd 100644 --- a/.github/workflows/lint-migrations.yml +++ b/.github/workflows/lint-migrations.yml @@ -16,7 +16,8 @@ jobs: run: | modified_migrations=$(git diff --diff-filter=d --name-only main 'packages/db/migrations/*.do.*.sql') echo "$modified_migrations" - echo "{FILE_NAMES}={$modified_migrations}" >> $GITHUB_OUTPUT + echo "file_names=$modified_migrations" >> $GITHUB_OUTPUT + id: modified-migrations - uses: sbdchd/squawk-action@v1 with: - pattern: "$FILE_NAMES" + pattern: ${{ steps.modified-migrations.outputs.file_names }} From 538eb382175ce024e0432aa587cbaf7b4e11432a Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 18:06:12 +0800 Subject: [PATCH 29/32] more testing --- packages/db/migrations/0183.do.test.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/db/migrations/0183.do.test.sql b/packages/db/migrations/0183.do.test.sql index 872c82fbe..5ac85ac63 100755 --- a/packages/db/migrations/0183.do.test.sql +++ b/packages/db/migrations/0183.do.test.sql @@ -11,6 +11,4 @@ CREATE TABLE omnivore.test ( updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP ); -CREATE TRIGGER update_test_modtime BEFORE UPDATE ON omnivore.test FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); - COMMIT; From d0b790f48b5b04972b927ad0c7d48888cf2c3e45 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 18:09:30 +0800 Subject: [PATCH 30/32] more testing --- packages/db/migrations/0183.do.test.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/db/migrations/0183.do.test.sql b/packages/db/migrations/0183.do.test.sql index 5ac85ac63..b0ab4eeea 100755 --- a/packages/db/migrations/0183.do.test.sql +++ b/packages/db/migrations/0183.do.test.sql @@ -11,4 +11,6 @@ CREATE TABLE omnivore.test ( updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP ); +CREATE INDEX test_name_idx ON omnivore.test (name); + COMMIT; From 95f3725dfa4b35f21aa0067dfadb767a36fc2add Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 18:11:07 +0800 Subject: [PATCH 31/32] remove testing sql --- packages/db/migrations/0183.do.test.sql | 16 ---------------- packages/db/migrations/0183.undo.test.sql | 7 ------- 2 files changed, 23 deletions(-) delete mode 100755 packages/db/migrations/0183.do.test.sql delete mode 100755 packages/db/migrations/0183.undo.test.sql diff --git a/packages/db/migrations/0183.do.test.sql b/packages/db/migrations/0183.do.test.sql deleted file mode 100755 index b0ab4eeea..000000000 --- a/packages/db/migrations/0183.do.test.sql +++ /dev/null @@ -1,16 +0,0 @@ --- Type: DO --- Name: test --- Description: test - -BEGIN; - -CREATE TABLE omnivore.test ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v1mc(), - name TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP -); - -CREATE INDEX test_name_idx ON omnivore.test (name); - -COMMIT; diff --git a/packages/db/migrations/0183.undo.test.sql b/packages/db/migrations/0183.undo.test.sql deleted file mode 100755 index 811a356c3..000000000 --- a/packages/db/migrations/0183.undo.test.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Type: UNDO --- Name: test --- Description: test - -BEGIN; - -COMMIT; From 0c87e465783b4c7ee084af4e8f4ec6fe3166db41 Mon Sep 17 00:00:00 2001 From: Hongbo Wu <hongbo@omnivore.app> Date: Fri, 21 Jun 2024 19:40:22 +0800 Subject: [PATCH 32/32] fix permission error when drop role omnivore_admin --- .../db/migrations/0183.do.alter_omnivore_admin_role.sql | 3 +-- .../db/migrations/0183.undo.alter_omnivore_admin_role.sql | 7 ++++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/db/migrations/0183.do.alter_omnivore_admin_role.sql b/packages/db/migrations/0183.do.alter_omnivore_admin_role.sql index 5a87699eb..ab74e309e 100755 --- a/packages/db/migrations/0183.do.alter_omnivore_admin_role.sql +++ b/packages/db/migrations/0183.do.alter_omnivore_admin_role.sql @@ -5,12 +5,11 @@ BEGIN; DROP POLICY user_admin_policy ON omnivore.user; +DROP POLICY library_item_admin_policy ON omnivore.library_item; REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA omnivore from omnivore_admin; REVOKE ALL PRIVILEGES ON SCHEMA omnivore from omnivore_admin; -DROP OWNED BY omnivore_admin; - DROP ROLE omnivore_admin; CREATE ROLE omnivore_admin; diff --git a/packages/db/migrations/0183.undo.alter_omnivore_admin_role.sql b/packages/db/migrations/0183.undo.alter_omnivore_admin_role.sql index 0b8c5fa6e..2299875d5 100755 --- a/packages/db/migrations/0183.undo.alter_omnivore_admin_role.sql +++ b/packages/db/migrations/0183.undo.alter_omnivore_admin_role.sql @@ -10,7 +10,7 @@ REVOKE SELECT, INSERT, UPDATE, DELETE ON omnivore.library_item FROM omnivore_adm DROP POLICY user_admin_policy ON omnivore.user; REVOKE SELECT, INSERT, UPDATE, DELETE ON omnivore.user FROM omnivore_admin; -DROP OWNED BY omnivore_admin; +REVOKE USAGE ON SCHEMA omnivore FROM omnivore_admin; DROP ROLE omnivore_admin; @@ -28,4 +28,9 @@ CREATE POLICY user_admin_policy on omnivore.user TO omnivore_admin USING (true); +CREATE POLICY library_item_admin_policy on omnivore.library_item + FOR ALL + TO omnivore_admin + USING (true); + COMMIT;