From 3cb93283b0e4a14f5128fa39e33f5e15377dab25 Mon Sep 17 00:00:00 2001 From: Hongbo Wu Date: Mon, 27 May 2024 11:37:59 +0800 Subject: [PATCH] add score library item job --- packages/api/src/jobs/score_library_item.ts | 80 +++++++++++++++++++ .../api/src/jobs/update_just_read_feed.ts | 45 +---------- packages/api/src/services/score.ts | 41 ++++++++++ packages/api/src/utils/helpers.ts | 5 ++ 4 files changed, 130 insertions(+), 41 deletions(-) create mode 100644 packages/api/src/jobs/score_library_item.ts create mode 100644 packages/api/src/services/score.ts diff --git a/packages/api/src/jobs/score_library_item.ts b/packages/api/src/jobs/score_library_item.ts new file mode 100644 index 000000000..5708316c6 --- /dev/null +++ b/packages/api/src/jobs/score_library_item.ts @@ -0,0 +1,80 @@ +import { + findLibraryItemById, + updateLibraryItem, +} from '../services/library_item' +import { Feature, getScores } from '../services/score' +import { lanaugeToCode } from '../utils/helpers' +import { logger } from '../utils/logger' + +export const SCORE_LIBRARY_ITEM_JOB = 'SCORE_LIBRARY_ITEM_JOB' + +export interface ScoreLibraryItemJobData { + userId: string + libraryItemId: string +} + +export const scoreLibraryItem = async ( + data: ScoreLibraryItemJobData +): Promise => { + logger.info('Scoring library item', data) + + const { userId, libraryItemId } = data + + const libraryItem = await findLibraryItemById(libraryItemId, userId, { + select: [ + 'id', + 'title', + 'thumbnail', + 'siteIcon', + 'savedAt', + 'siteName', + 'directionality', + 'folder', + 'author', + 'itemLanguage', + 'wordCount', + ], + }) + if (!libraryItem) { + logger.error('Library item not found', data) + return + } + + const itemFeatures = { + [libraryItem.id]: { + title: libraryItem.title, + has_thumbnail: !!libraryItem.thumbnail, + has_site_icon: !!libraryItem.siteIcon, + saved_at: libraryItem.savedAt, + site: libraryItem.siteName, + directionality: libraryItem.directionality, + folder: libraryItem.folder, + subscription_type: 'library', + author: libraryItem.author, + language: lanaugeToCode(libraryItem.itemLanguage || 'English'), + word_count: libraryItem.wordCount, + } as Feature, + } + + const scores = await getScores({ + user_id: userId, + item_features: itemFeatures, + }) + + logger.info('Scores', scores) + const score = scores[libraryItem.id] + if (!score) { + logger.error('Failed to score library item', data) + throw new Error('Failed to score library item') + } + + await updateLibraryItem( + libraryItem.id, + { + score, + }, + userId, + undefined, + true + ) +} diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts index 3f75ae8ad..85eca3d68 100644 --- a/packages/api/src/jobs/update_just_read_feed.ts +++ b/packages/api/src/jobs/update_just_read_feed.ts @@ -1,4 +1,3 @@ -import languages from '@cospired/i18n-iso-languages' import { LibraryItem } from '../entity/library_item' import { PublicItem } from '../entity/public_item' import { User } from '../entity/user' @@ -6,7 +5,9 @@ import { JustReadFeedSection } from '../generated/graphql' import { redisDataSource } from '../redis_data_source' import { findUnseenPublicItems } from '../services/just_read_feed' import { searchLibraryItems } from '../services/library_item' +import { Feature, getScores } from '../services/score' import { findActiveUser } from '../services/user' +import { lanaugeToCode } from '../utils/helpers' import { logger } from '../utils/logger' export const UPDATE_JUST_READ_FEED_JOB = 'UPDATE_JUST_READ_FEED_JOB' @@ -15,25 +16,6 @@ export interface UpdateJustReadFeedJobData { userId: string } -interface Feature { - title: string - has_thumbnail: boolean - has_site_icon: boolean - saved_at: Date - site?: string - language?: string - author?: string - directionality: string - word_count?: number - subscription_type: string - folder: string -} - -interface ScoreApiRequestBody { - user_id: string - item_features: Record // item_id -> feature -} - interface FeedItem { id: string title: string @@ -60,9 +42,6 @@ interface FeedItem { } } -const lanaugeToCode = (language: string): string => - languages.getAlpha2Code(language, 'en') || 'en' - const libraryItemToCandidate = (user: User, item: LibraryItem): FeedItem => ({ id: item.id, title: item.title, @@ -111,8 +90,6 @@ const publicItemToCandidate = (item: PublicItem): FeedItem => ({ }, }) -type ScoreApiResponse = Record // item_id -> score - const selectCandidates = async (user: User): Promise> => { const userId = user.id // get last 100 library items saved and not seen by user @@ -163,9 +140,7 @@ const rankCandidates = async ( return candidates } - // TODO: get score of candidates - const API_URL = 'http://127.0.0.1:5000/predictions' - const requestBody: ScoreApiRequestBody = { + const data = { user_id: userId, item_features: candidates.reduce((acc, item) => { acc[item.id] = { @@ -185,19 +160,7 @@ const rankCandidates = async ( }, {} as Record), } - const response = await fetch(API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - }) - - if (!response.ok) { - throw new Error(`Failed to score candidates: ${response.statusText}`) - } - - const scores = (await response.json()) as ScoreApiResponse + const scores = await getScores(data) // rank candidates by score in ascending order candidates.sort((a, b) => { diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts new file mode 100644 index 000000000..8e9d2785c --- /dev/null +++ b/packages/api/src/services/score.ts @@ -0,0 +1,41 @@ +export interface Feature { + title: string + has_thumbnail: boolean + has_site_icon: boolean + saved_at: Date + site?: string + language?: string + author?: string + directionality: string + word_count?: number + subscription_type: string + folder: string +} + +export interface ScoreApiRequestBody { + user_id: string + item_features: Record // item_id -> feature +} + +export type ScoreApiResponse = Record // item_id -> score + +export const getScores = async ( + data: ScoreApiRequestBody +): Promise => { + const API_URL = 'http://127.0.0.1:5000/predictions' + + const response = await fetch(API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }) + + if (!response.ok) { + throw new Error(`Failed to score candidates: ${response.statusText}`) + } + + const scores = (await response.json()) as ScoreApiResponse + return scores +} diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts index 01aaf9e94..30a821a5e 100644 --- a/packages/api/src/utils/helpers.ts +++ b/packages/api/src/utils/helpers.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import languages from '@cospired/i18n-iso-languages' import crypto from 'crypto' import Redis from 'ioredis' import normalizeUrl from 'normalize-url' @@ -31,6 +32,7 @@ import { validateUrl } from '../services/create_page_save_request' import { updateLibraryItem } from '../services/library_item' import { Merge } from '../util' import { logger } from './logger' + interface InputObject { // eslint-disable-next-line @typescript-eslint/no-explicit-any [key: string]: any @@ -423,3 +425,6 @@ export const getClientFromUserAgent = (userAgent: string): string => { return 'other' } + +export const lanaugeToCode = (language: string): string => + languages.getAlpha2Code(language, 'en') || 'en'