prehistoric-systems/omnivore
1
0
Fork 0
mirror of https://github.com/omnivore-app/omnivore.git synced 2026-03-11 08:54:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #4005 from omnivore-app/feature/refresh-home-api

Browse source 
feature/refresh home api
This commit is contained in:
Hongbo Wu 2024-05-29 17:37:41 +08:00 committed by GitHub
commit 69c555b8c3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 189 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
packages/api/src/generated/graphql.ts
View file

@ -279,6 +279,7 @@ export enum BulkActionType {
Archive = 'ARCHIVE',
Delete = 'DELETE',
MarkAsRead = 'MARK_AS_READ',
MarkAsSeen = 'MARK_AS_SEEN',
MoveToFolder = 'MOVE_TO_FOLDER'
}
@ -1713,6 +1714,7 @@ export type Mutation = {
optInFeature: OptInFeatureResult;
recommend: RecommendResult;
recommendHighlights: RecommendHighlightsResult;
refreshHome: RefreshHomeResult;
replyToEmail: ReplyToEmailResult;
reportItem: ReportItemResult;
revokeApiKey: RevokeApiKeyResult;
@ -2524,6 +2526,22 @@ export type RecommendingUser = {
username: Scalars['String'];
};
export type RefreshHomeError = {
__typename?: 'RefreshHomeError';
errorCodes: Array<RefreshHomeErrorCode>;
};
export enum RefreshHomeErrorCode {
Pending = 'PENDING'
}
export type RefreshHomeResult = RefreshHomeError | RefreshHomeSuccess;
export type RefreshHomeSuccess = {
__typename?: 'RefreshHomeSuccess';
success: Scalars['Boolean'];
};
export type Reminder = {
__typename?: 'Reminder';
archiveUntil: Scalars['Boolean'];
@ -2874,6 +2892,7 @@ export type SearchItem = {
recommendations?: Maybe<Array<Recommendation>>;
savedAt: Scalars['Date'];
score?: Maybe<Scalars['Float']>;
seenAt?: Maybe<Scalars['Date']>;
shortId?: Maybe<Scalars['String']>;
siteIcon?: Maybe<Scalars['String']>;
siteName?: Maybe<Scalars['String']>;
@ -4410,6 +4429,10 @@ export type ResolversTypes = {
Recommendation: ResolverTypeWrapper<Recommendation>;
RecommendationGroup: ResolverTypeWrapper<RecommendationGroup>;
RecommendingUser: ResolverTypeWrapper<RecommendingUser>;
RefreshHomeError: ResolverTypeWrapper<RefreshHomeError>;
RefreshHomeErrorCode: RefreshHomeErrorCode;
RefreshHomeResult: ResolversTypes['RefreshHomeError'] | ResolversTypes['RefreshHomeSuccess'];
RefreshHomeSuccess: ResolverTypeWrapper<RefreshHomeSuccess>;
Reminder: ResolverTypeWrapper<Reminder>;
ReminderError: ResolverTypeWrapper<ReminderError>;
ReminderErrorCode: ReminderErrorCode;
@ -4946,6 +4969,9 @@ export type ResolversParentTypes = {
Recommendation: Recommendation;
RecommendationGroup: RecommendationGroup;
RecommendingUser: RecommendingUser;
RefreshHomeError: RefreshHomeError;
RefreshHomeResult: ResolversParentTypes['RefreshHomeError'] | ResolversParentTypes['RefreshHomeSuccess'];
RefreshHomeSuccess: RefreshHomeSuccess;
Reminder: Reminder;
ReminderError: ReminderError;
ReminderResult: ResolversParentTypes['ReminderError'] | ResolversParentTypes['ReminderSuccess'];
@ -6390,6 +6416,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
optInFeature?: Resolver<ResolversTypes['OptInFeatureResult'], ParentType, ContextType, RequireFields<MutationOptInFeatureArgs, 'input'>>;
recommend?: Resolver<ResolversTypes['RecommendResult'], ParentType, ContextType, RequireFields<MutationRecommendArgs, 'input'>>;
recommendHighlights?: Resolver<ResolversTypes['RecommendHighlightsResult'], ParentType, ContextType, RequireFields<MutationRecommendHighlightsArgs, 'input'>>;
refreshHome?: Resolver<ResolversTypes['RefreshHomeResult'], ParentType, ContextType>;
replyToEmail?: Resolver<ResolversTypes['ReplyToEmailResult'], ParentType, ContextType, RequireFields<MutationReplyToEmailArgs, 'recentEmailId' | 'reply'>>;
reportItem?: Resolver<ResolversTypes['ReportItemResult'], ParentType, ContextType, RequireFields<MutationReportItemArgs, 'input'>>;
revokeApiKey?: Resolver<ResolversTypes['RevokeApiKeyResult'], ParentType, ContextType, RequireFields<MutationRevokeApiKeyArgs, 'id'>>;
@ -6662,6 +6689,20 @@ export type RecommendingUserResolvers<ContextType = ResolverContext, ParentType
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RefreshHomeErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RefreshHomeError'] = ResolversParentTypes['RefreshHomeError']> = {
errorCodes?: Resolver<Array<ResolversTypes['RefreshHomeErrorCode']>, ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type RefreshHomeResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RefreshHomeResult'] = ResolversParentTypes['RefreshHomeResult']> = {
__resolveType: TypeResolveFn<'RefreshHomeError' | 'RefreshHomeSuccess', ParentType, ContextType>;
};
export type RefreshHomeSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['RefreshHomeSuccess'] = ResolversParentTypes['RefreshHomeSuccess']> = {
success?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
__isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
};
export type ReminderResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Reminder'] = ResolversParentTypes['Reminder']> = {
archiveUntil?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
@ -6863,6 +6904,7 @@ export type SearchItemResolvers<ContextType = ResolverContext, ParentType extend
recommendations?: Resolver<Maybe<Array<ResolversTypes['Recommendation']>>, ParentType, ContextType>;
savedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
score?: Resolver<Maybe<ResolversTypes['Float']>, ParentType, ContextType>;
seenAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
shortId?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
siteIcon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
siteName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
@ -7795,6 +7837,9 @@ export type Resolvers<ContextType = ResolverContext> = {
Recommendation?: RecommendationResolvers<ContextType>;
RecommendationGroup?: RecommendationGroupResolvers<ContextType>;
RecommendingUser?: RecommendingUserResolvers<ContextType>;
RefreshHomeError?: RefreshHomeErrorResolvers<ContextType>;
RefreshHomeResult?: RefreshHomeResultResolvers<ContextType>;
RefreshHomeSuccess?: RefreshHomeSuccessResolvers<ContextType>;
Reminder?: ReminderResolvers<ContextType>;
ReminderError?: ReminderErrorResolvers<ContextType>;
ReminderResult?: ReminderResultResolvers<ContextType>;

17
packages/api/src/generated/schema.graphql
View file

@ -241,6 +241,7 @@ enum BulkActionType {
ARCHIVE
DELETE
MARK_AS_READ
MARK_AS_SEEN
MOVE_TO_FOLDER
}
@ -1550,6 +1551,7 @@ type Mutation {
optInFeature(input: OptInFeatureInput!): OptInFeatureResult!
recommend(input: RecommendInput!): RecommendResult!
recommendHighlights(input: RecommendHighlightsInput!): RecommendHighlightsResult!
refreshHome: RefreshHomeResult!
replyToEmail(recentEmailId: ID!, reply: AllowedReply!): ReplyToEmailResult!
reportItem(input: ReportItemInput!): ReportItemResult!
revokeApiKey(id: ID!): RevokeApiKeyResult!
@ -1890,6 +1892,20 @@ type RecommendingUser {
username: String!
}
type RefreshHomeError {
errorCodes: [RefreshHomeErrorCode!]!
}
enum RefreshHomeErrorCode {
PENDING
}
union RefreshHomeResult = RefreshHomeError | RefreshHomeSuccess
type RefreshHomeSuccess {
success: Boolean!
}
type Reminder {
archiveUntil: Boolean!
id: ID!
@ -2216,6 +2232,7 @@ type SearchItem {
recommendations: [Recommendation!]
savedAt: Date!
score: Float
seenAt: Date
shortId: String
siteIcon: String
siteName: String

25
packages/api/src/jobs/update_home.ts
View file

@ -235,6 +235,17 @@ export const getHomeSections = async (
return sections
}
export const deleteHome = async (userId: string) => {
const redisClient = redisDataSource.redisClient
if (!redisClient) {
throw new Error('Redis client not available')
}
const key = redisKey(userId)
await redisClient.del(key)
}
const appendSectionsToHome = async (
userId: string,
sections: Array<Section>,
@ -250,23 +261,25 @@ const appendSectionsToHome = async (
// store candidates in redis sorted set
const pipeline = redisClient.pipeline()
// sections expire in 24 hours
const ttl = 86_400_000
const now = Date.now()
const batchSize = sections.length
const savedAt = cursor ? cursor - batchSize - ttl : Date.now()
const savedAt = cursor ? cursor - batchSize : now
const scoreMembers = sections.flatMap((section, index) => [
savedAt + index + ttl, // score for the section is the savedAt + index + ttl
savedAt + index, // score for the section is the savedAt + index otherwise it will be the same for all sections
JSON.stringify(section),
])
// add section to the sorted set
pipeline.zadd(key, ...scoreMembers)
// remove expired sections and keep only the top 500
// remove expired sections and sections expire in 24 hours
const ttl = 86_400_000
pipeline.zremrangebyscore(key, '-inf', Date.now() - ttl)
// keep only the top MAX_FEED_ITEMS items
pipeline.zremrangebyrank(key, 0, -(MAX_FEED_ITEMS + 1))
pipeline.zremrangebyscore(key, '-inf', Date.now())
logger.info('Adding home sections to redis')
await pipeline.exec()

2
packages/api/src/resolvers/article/index.ts
View file

@ -670,7 +670,7 @@ export const searchResolver = authorized<
SearchSuccess,
SearchError,
QuerySearchArgs
>(async (_obj, params, { uid, log }) => {
>(async (_obj, params, { uid }) => {
const startCursor = params.after || ''
const first = Math.min(params.first || 10, 100) // limit to 100 items

4
packages/api/src/resolvers/function_resolvers.ts
View file

@ -60,7 +60,7 @@ import {
saveDiscoverArticleResolver,
} from './discover_feeds'
import { optInFeatureResolver } from './features'
import { homeResolver } from './home'
import { homeResolver, refreshHomeResolver } from './home'
import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
import {
addPopularReadResolver,
@ -332,6 +332,7 @@ export const functionResolvers = {
fetchContent: fetchContentResolver,
exportToIntegration: exportToIntegrationResolver,
replyToEmail: replyToEmailResolver,
refreshHome: refreshHomeResolver,
},
Query: {
me: getMeUserResolver,
@ -867,4 +868,5 @@ export const functionResolvers = {
...resultResolveTypeResolver('ReplyToEmail'),
...resultResolveTypeResolver('Home'),
...resultResolveTypeResolver('Subscription'),
...resultResolveTypeResolver('RefreshHome'),
}

42
packages/api/src/resolvers/home/index.ts
View file

@ -5,8 +5,11 @@ import {
HomeSection,
HomeSuccess,
QueryHomeArgs,
RefreshHomeError,
RefreshHomeErrorCode,
RefreshHomeSuccess,
} from '../../generated/graphql'
import { getHomeSections } from '../../jobs/update_home'
import { deleteHome, getHomeSections } from '../../jobs/update_home'
import { getJob } from '../../queue-processor'
import { Merge } from '../../util'
import { enqueueUpdateHomeJob, updateHomeJobId } from '../../utils/createTask'
@ -29,15 +32,17 @@ export const homeResolver = authorized<
QueryHomeArgs
>(async (_, { first, after }, { uid, log }) => {
const limit = first || 6
const cursor = after ? parseInt(after) : undefined
// cursor is the timestamp of the last item in the feed
// if cursor is not provided, it defaults to the current time
const cursor = after ? parseInt(after) : Date.now()
const sections = await getHomeSections(uid, limit, cursor)
log.info('Just read feed sections fetched')
log.info('Home sections fetched')
if (sections.length === 0) {
const existingJob = await getJob(updateHomeJobId(uid))
if (existingJob) {
log.info('Just read feed update job already enqueued')
log.info('Update job job already enqueued')
return {
errorCodes: [HomeErrorCode.Pending],
@ -49,7 +54,7 @@ export const homeResolver = authorized<
cursor,
})
log.info('Just read feed update enqueued')
log.info('Update home job enqueued')
return {
errorCodes: [HomeErrorCode.Pending],
@ -73,3 +78,30 @@ export const homeResolver = authorized<
},
}
})
export const refreshHomeResolver = authorized<
RefreshHomeSuccess,
RefreshHomeError
>(async (_, __, { uid, log }) => {
await deleteHome(uid)
log.info('Home cache deleted')
const existingJob = await getJob(updateHomeJobId(uid))
if (existingJob) {
log.info('Update home job already enqueued')
return {
errorCodes: [RefreshHomeErrorCode.Pending],
}
}
await enqueueUpdateHomeJob({
userId: uid,
})
log.info('Update home job enqueued')
return {
success: true,
}
})

17
packages/api/src/schema.ts
View file

@ -1677,6 +1677,7 @@ const schema = gql`
directionality: DirectionalityType
format: String
score: Float
seenAt: Date
}
type SearchItemEdge {
@ -2595,6 +2596,7 @@ const schema = gql`
MARK_AS_READ
ADD_LABELS
MOVE_TO_FOLDER
MARK_AS_SEEN
}
union BulkActionResult = BulkActionSuccess | BulkActionError
@ -3192,6 +3194,20 @@ const schema = gql`
union SubscriptionResult = SubscriptionSuccess | SubscriptionError
type RefreshHomeSuccess {
success: Boolean!
}
enum RefreshHomeErrorCode {
PENDING
}
type RefreshHomeError {
errorCodes: [RefreshHomeErrorCode!]!
}
union RefreshHomeResult = RefreshHomeSuccess | RefreshHomeError
# Mutations
type Mutation {
googleLogin(input: GoogleLoginInput!): LoginResult!
@ -3317,6 +3333,7 @@ const schema = gql`
): DeleteDiscoverFeedResult!
editDiscoverFeed(input: EditDiscoverFeedInput!): EditDiscoverFeedResult!
emptyTrash: EmptyTrashResult!
refreshHome: RefreshHomeResult!
}
# FIXME: remove sort from feedArticles after all cached tabs are closed

5
packages/api/src/services/library_item.ts
View file

@ -1303,6 +1303,11 @@ export const batchUpdateLibraryItems = async (
savedAt: now,
}
break
case BulkActionType.MarkAsSeen:
values = {
seenAt: now,
}
break
default:
throw new Error('Invalid bulk action')

45
packages/api/test/resolvers/article.test.ts
View file

@ -2322,6 +2322,51 @@ describe('Article API', () => {
expect(response.body.data.search.pageInfo.totalCount).to.eql(0)
})
})
context(
'when action is MarkAsSeen and query contains a list of item id',
() => {
const items: LibraryItem[] = []
before(async () => {
// Create some test items
for (let i = 0; i < 5; i++) {
const item = await createOrUpdateLibraryItem(
{
user,
title: 'test item',
slug: '',
originalUrl: `https://blog.omnivore.app/p/bulk-action-${i}`,
},
user.id
)
items.push(item)
}
})
after(async () => {
// Delete all items
await deleteLibraryItemsByUserId(user.id)
})
it('marks items as seen', async () => {
const query = `includes:${items.map((i) => i.id).join(',')}`
const res = await graphqlRequest(
bulkActionQuery(BulkActionType.MarkAsSeen, query),
authToken
).expect(200)
expect(res.body.data.bulkAction.success).to.be.true
const response = await graphqlRequest(
searchQuery('is:seen'),
authToken
).expect(200)
expect(response.body.data.search.pageInfo.totalCount).to.eql(5)
})
}
)
})
describe('SetFavoriteArticle API', () => {