From 85ee10c1c56b9e2bebe1bde0a3b22a09208d88b6 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 21 May 2024 15:37:59 +0800
Subject: [PATCH 01/33] add tables for public inventory

---
 .../db/migrations/0177.do.public_item.sql     | 71 +++++++++++++++++++
 .../db/migrations/0177.undo.public_item.sql   | 13 ++++
 2 files changed, 84 insertions(+)
 create mode 100755 packages/db/migrations/0177.do.public_item.sql
 create mode 100755 packages/db/migrations/0177.undo.public_item.sql

diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
new file mode 100755
index 000000000..3ff7bcb03
--- /dev/null
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -0,0 +1,71 @@
+-- Type: DO
+-- Name: public_item
+-- Description: Create a table for public items
+
+BEGIN;
+
+CREATE TABLE omnivore.public_item_source (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    name TEXT NOT NULL,
+    topics TEXT[] NOT NULL,
+    thumbnail TEXT NOT NULL,
+    url TEXT NOT NULL,
+    language_codes TEXT[] NOT NULL,
+    created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE omnivore.public_item (
+	id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    source_id uuid NOT NULL, -- user_id or public_item_source_id
+    type TEXT NOT NULL, -- public feeds, newsletters, or user recommended
+    title TEXT NOT NULL,
+    url TEXT NOT NULL,
+    thumbnail TEXT,
+    preview_content TEXT,
+    language_code TEXT,
+    author TEXT,
+    dir TEXT,
+    published_at timestamptz,
+    created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE omnivore.public_item_features (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    public_item_id uuid NOT NULL REFERENCES omnivore.public_item(id) ON DELETE CASCADE,
+    classified_topic TEXT,
+    sentiment_score FLOAT,
+    writing_style TEXT,
+    popularity_score FLOAT,
+    embedding VECTOR(768),
+    created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX public_item_feature_public_item_id_idx ON omnivore.public_item_features(public_item_id);
+
+CREATE TABLE omnivore.public_item_stats (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    public_item_id uuid NOT NULL REFERENCES omnivore.public_item(id) ON DELETE CASCADE,
+    save_count INT NOT NULL DEFAULT 0,
+    like_count INT NOT NULL DEFAULT 0,
+    broadcast_count INT NOT NULL DEFAULT 0,
+    created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX public_item_stats_public_item_id_idx ON omnivore.public_item_stats(public_item_id);
+
+CREATE TABLE omnivore.public_item_interactions (
+	id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    user_id uuid NOT NULL REFERENCES omnivore.user(id) ON DELETE CASCADE,
+    public_item_id uuid NOT NULL REFERENCES omnivore.public_item(id) ON DELETE CASCADE,
+    action TEXT NOT NULL, -- save, like, broadcast, comment, see
+    action_data TEXT, -- for comment, the comment text
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX public_item_interaction_user_id_idx ON omnivore.public_item_interactions(user_id);
+CREATE INDEX public_item_interaction_public_item_id_idx ON omnivore.public_item_interactions(public_item_id);
+
+COMMIT;
diff --git a/packages/db/migrations/0177.undo.public_item.sql b/packages/db/migrations/0177.undo.public_item.sql
new file mode 100755
index 000000000..31ab4da9f
--- /dev/null
+++ b/packages/db/migrations/0177.undo.public_item.sql
@@ -0,0 +1,13 @@
+-- Type: UNDO
+-- Name: public_item
+-- Description: Create a table for public items
+
+BEGIN;
+
+DROP TABLE omnivore.public_item_interactions;
+DROP TABLE omnivore.public_item_stats;
+DROP TABLE omnivore.public_item_features;
+DROP TABLE omnivore.public_item;
+DROP TABLE omnivore.public_item_source;
+
+COMMIT;

From 912fc566a96db18c9e1f198dcb521e1a45fc028a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 23 May 2024 11:32:14 +0800
Subject: [PATCH 02/33] add approved column and seen_at

---
 packages/db/migrations/0177.do.public_item.sql | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index 3ff7bcb03..4d0a1a750 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -8,9 +8,10 @@ CREATE TABLE omnivore.public_item_source (
     id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
     name TEXT NOT NULL,
     topics TEXT[] NOT NULL,
-    thumbnail TEXT NOT NULL,
+    thumbnail TEXT,
     url TEXT NOT NULL,
     language_codes TEXT[] NOT NULL,
+    approved BOOLEAN NOT NULL DEFAULT FALSE,
     created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
     updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
@@ -21,6 +22,7 @@ CREATE TABLE omnivore.public_item (
     type TEXT NOT NULL, -- public feeds, newsletters, or user recommended
     title TEXT NOT NULL,
     url TEXT NOT NULL,
+    approved BOOLEAN NOT NULL DEFAULT FALSE,
     thumbnail TEXT,
     preview_content TEXT,
     language_code TEXT,
@@ -68,4 +70,15 @@ CREATE TABLE omnivore.public_item_interactions (
 CREATE INDEX public_item_interaction_user_id_idx ON omnivore.public_item_interactions(user_id);
 CREATE INDEX public_item_interaction_public_item_id_idx ON omnivore.public_item_interactions(public_item_id);
 
+CREATE TABLE omnivore.library_item_interactions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
+    user_id uuid NOT NULL REFERENCES omnivore.user(id) ON DELETE CASCADE,
+    library_item_id uuid NOT NULL REFERENCES omnivore.library_item(id) ON DELETE CASCADE,
+    action TEXT NOT NULL, -- seen
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX library_item_interaction_user_id_idx ON omnivore.library_item_interactions(user_id);
+CREATE INDEX library_item_interaction_library_item_id_idx ON omnivore.library_item_interactions(library_item_id);
+
 COMMIT;

From 43ebdc30a18a427f94a289c48c540f564e4addfe Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 23 May 2024 11:36:01 +0800
Subject: [PATCH 03/33] update just read api

---
 packages/api/src/generated/graphql.ts     | 140 ++++++++++++++++++++++
 packages/api/src/generated/schema.graphql |  54 +++++++++
 packages/api/src/schema.ts                |  55 +++++++++
 3 files changed, 249 insertions(+)

diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index aa96fbe3b..2eaa03ac2 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1363,6 +1363,64 @@ export type JoinGroupSuccess = {
   group: RecommendationGroup;
 };
 
+export type JustReadFeedError = {
+  __typename?: 'JustReadFeedError';
+  errorCodes: Array<JustReadFeedErrorCode>;
+};
+
+export enum JustReadFeedErrorCode {
+  BadRequest = 'BAD_REQUEST',
+  Unauthorized = 'UNAUTHORIZED'
+}
+
+export type JustReadFeedItem = {
+  __typename?: 'JustReadFeedItem';
+  author?: Maybe<Scalars['String']>;
+  broadcastCount?: Maybe<Scalars['Int']>;
+  comments?: Maybe<Array<Scalars['String']>>;
+  createdAt?: Maybe<Scalars['Date']>;
+  dir?: Maybe<Scalars['String']>;
+  highlights?: Maybe<Scalars['String']>;
+  id: Scalars['ID'];
+  languageCode?: Maybe<Scalars['String']>;
+  likedCount?: Maybe<Scalars['Int']>;
+  previewContent?: Maybe<Scalars['String']>;
+  publishedAt?: Maybe<Scalars['Date']>;
+  savedCount?: Maybe<Scalars['Int']>;
+  seen_at?: Maybe<Scalars['Date']>;
+  source?: Maybe<JustReadFeedSource>;
+  thumbnail?: Maybe<Scalars['String']>;
+  title: Scalars['String'];
+  topic: Scalars['String'];
+  type: Scalars['String'];
+  updatedAt?: Maybe<Scalars['Date']>;
+  url: Scalars['String'];
+};
+
+export type JustReadFeedResult = JustReadFeedError | JustReadFeedSuccess;
+
+export type JustReadFeedSource = {
+  __typename?: 'JustReadFeedSource';
+  id: Scalars['ID'];
+  languageCodes?: Maybe<Array<Scalars['String']>>;
+  name: Scalars['String'];
+  thumbnail?: Maybe<Scalars['String']>;
+  topics?: Maybe<Array<Scalars['String']>>;
+  url?: Maybe<Scalars['String']>;
+};
+
+export type JustReadFeedSuccess = {
+  __typename?: 'JustReadFeedSuccess';
+  topics: Array<JustReadFeedTopic>;
+};
+
+export type JustReadFeedTopic = {
+  __typename?: 'JustReadFeedTopic';
+  items: Array<JustReadFeedItem>;
+  name: Scalars['String'];
+  thumbnail?: Maybe<Scalars['String']>;
+};
+
 export type Label = {
   __typename?: 'Label';
   color: Scalars['String'];
@@ -2161,6 +2219,7 @@ export type Query = {
   hello?: Maybe<Scalars['String']>;
   integration: IntegrationResult;
   integrations: IntegrationsResult;
+  justReadFeed: JustReadFeedResult;
   labels: LabelsResult;
   me?: Maybe<User>;
   newsletterEmails: NewsletterEmailsResult;
@@ -2212,6 +2271,12 @@ export type QueryIntegrationArgs = {
 };
 
 
+export type QueryJustReadFeedArgs = {
+  language?: InputMaybe<Scalars['String']>;
+  location?: InputMaybe<Scalars['String']>;
+};
+
+
 export type QueryRulesArgs = {
   enabled?: InputMaybe<Scalars['Boolean']>;
 };
@@ -4201,6 +4266,13 @@ export type ResolversTypes = {
   JoinGroupErrorCode: JoinGroupErrorCode;
   JoinGroupResult: ResolversTypes['JoinGroupError'] | ResolversTypes['JoinGroupSuccess'];
   JoinGroupSuccess: ResolverTypeWrapper<JoinGroupSuccess>;
+  JustReadFeedError: ResolverTypeWrapper<JustReadFeedError>;
+  JustReadFeedErrorCode: JustReadFeedErrorCode;
+  JustReadFeedItem: ResolverTypeWrapper<JustReadFeedItem>;
+  JustReadFeedResult: ResolversTypes['JustReadFeedError'] | ResolversTypes['JustReadFeedSuccess'];
+  JustReadFeedSource: ResolverTypeWrapper<JustReadFeedSource>;
+  JustReadFeedSuccess: ResolverTypeWrapper<JustReadFeedSuccess>;
+  JustReadFeedTopic: ResolverTypeWrapper<JustReadFeedTopic>;
   Label: ResolverTypeWrapper<Label>;
   LabelsError: ResolverTypeWrapper<LabelsError>;
   LabelsErrorCode: LabelsErrorCode;
@@ -4743,6 +4815,12 @@ export type ResolversParentTypes = {
   JoinGroupError: JoinGroupError;
   JoinGroupResult: ResolversParentTypes['JoinGroupError'] | ResolversParentTypes['JoinGroupSuccess'];
   JoinGroupSuccess: JoinGroupSuccess;
+  JustReadFeedError: JustReadFeedError;
+  JustReadFeedItem: JustReadFeedItem;
+  JustReadFeedResult: ResolversParentTypes['JustReadFeedError'] | ResolversParentTypes['JustReadFeedSuccess'];
+  JustReadFeedSource: JustReadFeedSource;
+  JustReadFeedSuccess: JustReadFeedSuccess;
+  JustReadFeedTopic: JustReadFeedTopic;
   Label: Label;
   LabelsError: LabelsError;
   LabelsResult: ResolversParentTypes['LabelsError'] | ResolversParentTypes['LabelsSuccess'];
@@ -5988,6 +6066,61 @@ export type JoinGroupSuccessResolvers<ContextType = ResolverContext, ParentType
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type JustReadFeedErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedError'] = ResolversParentTypes['JustReadFeedError']> = {
+  errorCodes?: Resolver<Array<ResolversTypes['JustReadFeedErrorCode']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type JustReadFeedItemResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedItem'] = ResolversParentTypes['JustReadFeedItem']> = {
+  author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  broadcastCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  comments?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
+  createdAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  dir?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  highlights?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  languageCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  likedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  publishedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  savedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  source?: Resolver<Maybe<ResolversTypes['JustReadFeedSource']>, ParentType, ContextType>;
+  thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  topic?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  type?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  updatedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type JustReadFeedResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedResult'] = ResolversParentTypes['JustReadFeedResult']> = {
+  __resolveType: TypeResolveFn<'JustReadFeedError' | 'JustReadFeedSuccess', ParentType, ContextType>;
+};
+
+export type JustReadFeedSourceResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSource'] = ResolversParentTypes['JustReadFeedSource']> = {
+  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  languageCodes?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  topics?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
+  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type JustReadFeedSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSuccess'] = ResolversParentTypes['JustReadFeedSuccess']> = {
+  topics?: Resolver<Array<ResolversTypes['JustReadFeedTopic']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type JustReadFeedTopicResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedTopic'] = ResolversParentTypes['JustReadFeedTopic']> = {
+  items?: Resolver<Array<ResolversTypes['JustReadFeedItem']>, ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type LabelResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Label'] = ResolversParentTypes['Label']> = {
   color?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   createdAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
@@ -6315,6 +6448,7 @@ export type QueryResolvers<ContextType = ResolverContext, ParentType extends Res
   hello?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   integration?: Resolver<ResolversTypes['IntegrationResult'], ParentType, ContextType, RequireFields<QueryIntegrationArgs, 'name'>>;
   integrations?: Resolver<ResolversTypes['IntegrationsResult'], ParentType, ContextType>;
+  justReadFeed?: Resolver<ResolversTypes['JustReadFeedResult'], ParentType, ContextType, Partial<QueryJustReadFeedArgs>>;
   labels?: Resolver<ResolversTypes['LabelsResult'], ParentType, ContextType>;
   me?: Resolver<Maybe<ResolversTypes['User']>, ParentType, ContextType>;
   newsletterEmails?: Resolver<ResolversTypes['NewsletterEmailsResult'], ParentType, ContextType>;
@@ -7505,6 +7639,12 @@ export type Resolvers<ContextType = ResolverContext> = {
   JoinGroupError?: JoinGroupErrorResolvers<ContextType>;
   JoinGroupResult?: JoinGroupResultResolvers<ContextType>;
   JoinGroupSuccess?: JoinGroupSuccessResolvers<ContextType>;
+  JustReadFeedError?: JustReadFeedErrorResolvers<ContextType>;
+  JustReadFeedItem?: JustReadFeedItemResolvers<ContextType>;
+  JustReadFeedResult?: JustReadFeedResultResolvers<ContextType>;
+  JustReadFeedSource?: JustReadFeedSourceResolvers<ContextType>;
+  JustReadFeedSuccess?: JustReadFeedSuccessResolvers<ContextType>;
+  JustReadFeedTopic?: JustReadFeedTopicResolvers<ContextType>;
   Label?: LabelResolvers<ContextType>;
   LabelsError?: LabelsErrorResolvers<ContextType>;
   LabelsResult?: LabelsResultResolvers<ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 0b08658b0..696fb8b3e 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1222,6 +1222,59 @@ type JoinGroupSuccess {
   group: RecommendationGroup!
 }
 
+type JustReadFeedError {
+  errorCodes: [JustReadFeedErrorCode!]!
+}
+
+enum JustReadFeedErrorCode {
+  BAD_REQUEST
+  UNAUTHORIZED
+}
+
+type JustReadFeedItem {
+  author: String
+  broadcastCount: Int
+  comments: [String!]
+  createdAt: Date
+  dir: String
+  highlights: String
+  id: ID!
+  languageCode: String
+  likedCount: Int
+  previewContent: String
+  publishedAt: Date
+  savedCount: Int
+  seen_at: Date
+  source: JustReadFeedSource
+  thumbnail: String
+  title: String!
+  topic: String!
+  type: String!
+  updatedAt: Date
+  url: String!
+}
+
+union JustReadFeedResult = JustReadFeedError | JustReadFeedSuccess
+
+type JustReadFeedSource {
+  id: ID!
+  languageCodes: [String!]
+  name: String!
+  thumbnail: String
+  topics: [String!]
+  url: String
+}
+
+type JustReadFeedSuccess {
+  topics: [JustReadFeedTopic!]!
+}
+
+type JustReadFeedTopic {
+  items: [JustReadFeedItem!]!
+  name: String!
+  thumbnail: String
+}
+
 type Label {
   color: String!
   createdAt: Date
@@ -1640,6 +1693,7 @@ type Query {
   hello: String
   integration(name: String!): IntegrationResult!
   integrations: IntegrationsResult!
+  justReadFeed(language: String, location: String): JustReadFeedResult!
   labels: LabelsResult!
   me: User
   newsletterEmails: NewsletterEmailsResult!
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index a8002e17e..c6dd0ed50 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3101,6 +3101,60 @@ const schema = gql`
     SUBSCRIBE
   }
 
+  type JustReadFeedSource {
+    id: ID!
+    name: String!
+    url: String!
+    topics: [String!]
+    thumbnail: String
+    url: String
+    languageCodes: [String!]
+  }
+
+  type JustReadFeedItem {
+    id: ID!
+    title: String!
+    topic: String!
+    url: String!
+    thumbnail: String
+    publishedAt: Date
+    source: JustReadFeedSource
+    previewContent: String
+    highlights: String
+    savedCount: Int
+    likedCount: Int
+    broadcastCount: Int
+    createdAt: Date
+    updatedAt: Date
+    comments: [String!]
+    author: String
+    type: String!
+    languageCode: String
+    dir: String
+    seen_at: Date
+  }
+
+  type JustReadFeedTopic {
+    name: String!
+    items: [JustReadFeedItem!]!
+    thumbnail: String
+  }
+
+  type JustReadFeedSuccess {
+    topics: [JustReadFeedTopic!]!
+  }
+
+  enum JustReadFeedErrorCode {
+    UNAUTHORIZED
+    BAD_REQUEST
+  }
+
+  type JustReadFeedError {
+    errorCodes: [JustReadFeedErrorCode!]!
+  }
+
+  union JustReadFeedResult = JustReadFeedSuccess | JustReadFeedError
+
   # Mutations
   type Mutation {
     googleLogin(input: GoogleLoginInput!): LoginResult!
@@ -3296,6 +3350,7 @@ const schema = gql`
     feeds(input: FeedsInput!): FeedsResult!
     discoverFeeds: DiscoverFeedResult!
     scanFeeds(input: ScanFeedsInput!): ScanFeedsResult!
+    justReadFeed(location: String, language: String): JustReadFeedResult!
   }
 `
 

From 06b89a88b3086a3079fcf999510bf0a76b91e149 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 23 May 2024 15:48:30 +0800
Subject: [PATCH 04/33] update candidates selection

---
 packages/api/src/entity/library_item.ts       |  3 +
 packages/api/src/entity/public_item.ts        | 57 +++++++++++++++++++
 .../api/src/entity/public_item_interaction.ts | 42 ++++++++++++++
 .../api/src/jobs/update_just_read_feed.ts     | 40 +++++++++++++
 packages/api/src/services/library_item.ts     |  3 +
 packages/api/src/services/public_item.ts      | 28 +++++++++
 .../db/migrations/0177.do.public_item.sql     | 35 +++++-------
 .../db/migrations/0177.undo.public_item.sql   |  3 +-
 8 files changed, 188 insertions(+), 23 deletions(-)
 create mode 100644 packages/api/src/entity/public_item.ts
 create mode 100644 packages/api/src/entity/public_item_interaction.ts
 create mode 100644 packages/api/src/jobs/update_just_read_feed.ts
 create mode 100644 packages/api/src/services/public_item.ts

diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts
index fb92fa618..f6a2f83de 100644
--- a/packages/api/src/entity/library_item.ts
+++ b/packages/api/src/entity/library_item.ts
@@ -204,4 +204,7 @@ export class LibraryItem {
 
   @Column('text')
   highlightAnnotations?: string[]
+
+  @Column('timestamptz')
+  seenAt?: Date
 }
diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts
new file mode 100644
index 000000000..517c9fab6
--- /dev/null
+++ b/packages/api/src/entity/public_item.ts
@@ -0,0 +1,57 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  OneToOne,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm'
+import { PublicItemInteraction } from './public_item_interaction'
+
+@Entity()
+export class PublicItem {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @OneToOne(() => PublicItemInteraction)
+  interaction?: PublicItemInteraction
+
+  @Column('uuid')
+  sourceId!: string
+
+  @Column('text')
+  type!: string
+
+  @Column('text')
+  title!: string
+
+  @Column('text')
+  url!: string
+
+  @Column('boolean')
+  approved!: boolean
+
+  @Column('text', { nullable: true })
+  thumbnail?: string | null
+
+  @Column('text', { nullable: true })
+  previewContent?: string | null
+
+  @Column('text', { nullable: true })
+  languageCode?: string | null
+
+  @Column('text', { nullable: true })
+  author?: string | null
+
+  @Column('text', { nullable: true })
+  dir?: string | null
+
+  @Column('timestamptz', { nullable: true })
+  publishedAt?: Date | null
+
+  @CreateDateColumn()
+  createdAt!: Date
+
+  @UpdateDateColumn()
+  updatedAt!: Date
+}
diff --git a/packages/api/src/entity/public_item_interaction.ts b/packages/api/src/entity/public_item_interaction.ts
new file mode 100644
index 000000000..379962d4d
--- /dev/null
+++ b/packages/api/src/entity/public_item_interaction.ts
@@ -0,0 +1,42 @@
+import {
+  Column,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  OneToOne,
+  PrimaryGeneratedColumn,
+} from 'typeorm'
+import { PublicItem } from './public_item'
+import { User } from './user'
+
+@Entity({ name: 'public_item_interactions' })
+export class PublicItemInteraction {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @OneToOne(() => PublicItem, { onDelete: 'CASCADE' })
+  @JoinColumn({ name: 'public_item_id' })
+  publicItem!: PublicItem
+
+  @ManyToOne(() => User, { onDelete: 'CASCADE' })
+  @JoinColumn({ name: 'user_id' })
+  user!: User
+
+  @Column('timestamptz')
+  seenAt?: Date
+
+  @Column('timestamptz')
+  savedAt?: Date
+
+  @Column('timestamptz')
+  likedAt?: Date
+
+  @Column('timestamptz')
+  broadcastedAt?: Date
+
+  @Column('timestamptz')
+  createdAt!: Date
+
+  @Column('timestamptz')
+  updated!: Date
+}
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
new file mode 100644
index 000000000..7e22f007a
--- /dev/null
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -0,0 +1,40 @@
+import { searchLibraryItems } from '../services/library_item'
+import { findUnseenPublicItems } from '../services/public_item'
+import { logger } from '../utils/logger'
+
+interface JustReadFeedUpdateData {
+  userId: string
+}
+
+const selectCandidates = async (userId: string) => {
+  // get last 100 library items saved and not seen by user
+  const privateCandidates = await searchLibraryItems(
+    {
+      size: 100,
+      includeContent: false,
+      query: `-is:seen`,
+    },
+    userId
+  )
+
+  // get candidates from public inventory
+  const publicCandidates = await findUnseenPublicItems(userId, {
+    limit: 100,
+  })
+
+  // TODO: mix candidates
+  return privateCandidates.concat(publicCandidates)
+}
+
+const updateJustReadFeed = async (data: JustReadFeedUpdateData) => {
+  const { userId } = data
+  logger.info(`Updating just read feed for user ${userId}`)
+
+  const candidates = await selectCandidates(userId)
+  logger.info(`Found ${candidates.length} candidates`)
+
+  // TODO: integrity check on candidates?
+  // TODO: rank candidates
+
+  // TODO: prepend candidates to feed in redis
+}
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index b9e3aeb3e..03a66e97c 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -60,6 +60,7 @@ enum ReadFilter {
   READ = 'read',
   READING = 'reading',
   UNREAD = 'unread',
+  SEEN = 'seen',
 }
 
 enum InFilter {
@@ -332,6 +333,8 @@ export const buildQueryString = (
               return 'library_item.reading_progress_bottom_percent BETWEEN 2 AND 98'
             case ReadFilter.UNREAD:
               return 'library_item.reading_progress_bottom_percent < 2'
+            case ReadFilter.SEEN:
+              return 'library_item.seen_at IS NOT NULL'
             default:
               throw new Error(`Unexpected keyword: ${value}`)
           }
diff --git a/packages/api/src/services/public_item.ts b/packages/api/src/services/public_item.ts
new file mode 100644
index 000000000..58243abda
--- /dev/null
+++ b/packages/api/src/services/public_item.ts
@@ -0,0 +1,28 @@
+import { IsNull } from 'typeorm'
+import { PublicItem } from '../entity/public_item'
+import { getRepository } from '../repository'
+
+export const findUnseenPublicItems = async (
+  userId: string,
+  options: {
+    limit?: number
+    offset?: number
+  }
+) =>
+  getRepository(PublicItem).find({
+    where: {
+      interaction: IsNull(),
+      interaction: {
+        user: {
+          id: userId,
+        },
+        seenAt: IsNull(),
+      },
+      approved: true,
+    },
+    order: {
+      createdAt: 'DESC',
+    },
+    take: options.limit,
+    skip: options.offset,
+  })
diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index 4d0a1a750..c657dced7 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -16,6 +16,9 @@ CREATE TABLE omnivore.public_item_source (
     updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
+CREATE TRIGGER update_public_item_source_modtime BEFORE UPDATE ON omnivore.public_item_source FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
+
+
 CREATE TABLE omnivore.public_item (
 	id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
     source_id uuid NOT NULL, -- user_id or public_item_source_id
@@ -33,18 +36,8 @@ CREATE TABLE omnivore.public_item (
     updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
-CREATE TABLE omnivore.public_item_features (
-    id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
-    public_item_id uuid NOT NULL REFERENCES omnivore.public_item(id) ON DELETE CASCADE,
-    classified_topic TEXT,
-    sentiment_score FLOAT,
-    writing_style TEXT,
-    popularity_score FLOAT,
-    embedding VECTOR(768),
-    created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
-);
+CREATE TRIGGER update_public_item_modtime BEFORE UPDATE ON omnivore.public_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
 
-CREATE INDEX public_item_feature_public_item_id_idx ON omnivore.public_item_features(public_item_id);
 
 CREATE TABLE omnivore.public_item_stats (
     id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
@@ -57,28 +50,26 @@ CREATE TABLE omnivore.public_item_stats (
 );
 
 CREATE INDEX public_item_stats_public_item_id_idx ON omnivore.public_item_stats(public_item_id);
+CREATE TRIGGER update_public_item_stats_modtime BEFORE UPDATE ON omnivore.public_item_stats FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
+
 
 CREATE TABLE omnivore.public_item_interactions (
 	id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
     user_id uuid NOT NULL REFERENCES omnivore.user(id) ON DELETE CASCADE,
     public_item_id uuid NOT NULL REFERENCES omnivore.public_item(id) ON DELETE CASCADE,
-    action TEXT NOT NULL, -- save, like, broadcast, comment, see
-    action_data TEXT, -- for comment, the comment text
+    saved_at TIMESTAMPTZ,
+    liked_at TIMESTAMPTZ,
+    broadcasted_at TIMESTAMPTZ,
+    seen_at TIMESTAMPTZ,
     created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
 CREATE INDEX public_item_interaction_user_id_idx ON omnivore.public_item_interactions(user_id);
 CREATE INDEX public_item_interaction_public_item_id_idx ON omnivore.public_item_interactions(public_item_id);
+CREATE TRIGGER update_public_item_interactions_modtime BEFORE UPDATE ON omnivore.public_item_interactions FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
 
-CREATE TABLE omnivore.library_item_interactions (
-    id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
-    user_id uuid NOT NULL REFERENCES omnivore.user(id) ON DELETE CASCADE,
-    library_item_id uuid NOT NULL REFERENCES omnivore.library_item(id) ON DELETE CASCADE,
-    action TEXT NOT NULL, -- seen
-    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
-);
 
-CREATE INDEX library_item_interaction_user_id_idx ON omnivore.library_item_interactions(user_id);
-CREATE INDEX library_item_interaction_library_item_id_idx ON omnivore.library_item_interactions(library_item_id);
+ALTER TABLE omnivore.library_item ADD COLUMN seen_at timestamptz;
 
 COMMIT;
diff --git a/packages/db/migrations/0177.undo.public_item.sql b/packages/db/migrations/0177.undo.public_item.sql
index 31ab4da9f..36c5f8165 100755
--- a/packages/db/migrations/0177.undo.public_item.sql
+++ b/packages/db/migrations/0177.undo.public_item.sql
@@ -6,8 +6,9 @@ BEGIN;
 
 DROP TABLE omnivore.public_item_interactions;
 DROP TABLE omnivore.public_item_stats;
-DROP TABLE omnivore.public_item_features;
 DROP TABLE omnivore.public_item;
 DROP TABLE omnivore.public_item_source;
 
+ALTER TABLE omnivore.library_item DROP COLUMN seen_at;
+
 COMMIT;

From b058952c2d0609328a30e690fe210e0c30b4a515 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 23 May 2024 21:10:01 +0800
Subject: [PATCH 05/33] implement updateJustReadFeed job

---
 packages/api/src/entity/public_item.ts        |  34 +++---
 .../api/src/entity/public_item_interaction.ts |   8 +-
 .../api/src/jobs/update_just_read_feed.ts     | 109 ++++++++++++++++--
 packages/api/src/services/public_item.ts      |  33 +++---
 .../db/migrations/0177.do.public_item.sql     |   5 +-
 5 files changed, 140 insertions(+), 49 deletions(-)

diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts
index 517c9fab6..00554c3a8 100644
--- a/packages/api/src/entity/public_item.ts
+++ b/packages/api/src/entity/public_item.ts
@@ -2,22 +2,20 @@ import {
   Column,
   CreateDateColumn,
   Entity,
-  OneToOne,
   PrimaryGeneratedColumn,
   UpdateDateColumn,
 } from 'typeorm'
-import { PublicItemInteraction } from './public_item_interaction'
 
 @Entity()
 export class PublicItem {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 
-  @OneToOne(() => PublicItemInteraction)
-  interaction?: PublicItemInteraction
+  @Column('text')
+  source_name!: string
 
-  @Column('uuid')
-  sourceId!: string
+  @Column('text')
+  source_icon!: string
 
   @Column('text')
   type!: string
@@ -31,23 +29,23 @@ export class PublicItem {
   @Column('boolean')
   approved!: boolean
 
-  @Column('text', { nullable: true })
-  thumbnail?: string | null
+  @Column('text')
+  thumbnail?: string
 
-  @Column('text', { nullable: true })
-  previewContent?: string | null
+  @Column('text')
+  previewContent?: string
 
-  @Column('text', { nullable: true })
-  languageCode?: string | null
+  @Column('text')
+  languageCode?: string
 
-  @Column('text', { nullable: true })
-  author?: string | null
+  @Column('text')
+  author?: string
 
-  @Column('text', { nullable: true })
-  dir?: string | null
+  @Column('text')
+  dir?: string
 
-  @Column('timestamptz', { nullable: true })
-  publishedAt?: Date | null
+  @Column('timestamptz')
+  publishedAt?: Date
 
   @CreateDateColumn()
   createdAt!: Date
diff --git a/packages/api/src/entity/public_item_interaction.ts b/packages/api/src/entity/public_item_interaction.ts
index 379962d4d..efb370d58 100644
--- a/packages/api/src/entity/public_item_interaction.ts
+++ b/packages/api/src/entity/public_item_interaction.ts
@@ -3,7 +3,6 @@ import {
   Entity,
   JoinColumn,
   ManyToOne,
-  OneToOne,
   PrimaryGeneratedColumn,
 } from 'typeorm'
 import { PublicItem } from './public_item'
@@ -14,16 +13,19 @@ export class PublicItemInteraction {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 
-  @OneToOne(() => PublicItem, { onDelete: 'CASCADE' })
+  @ManyToOne(() => PublicItem, { onDelete: 'CASCADE' })
   @JoinColumn({ name: 'public_item_id' })
   publicItem!: PublicItem
 
+  @Column('uuid')
+  publicItemId!: string
+
   @ManyToOne(() => User, { onDelete: 'CASCADE' })
   @JoinColumn({ name: 'user_id' })
   user!: User
 
   @Column('timestamptz')
-  seenAt?: Date
+  seenAt!: Date
 
   @Column('timestamptz')
   savedAt?: Date
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 7e22f007a..413dd961c 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -1,3 +1,6 @@
+import { LibraryItem } from '../entity/library_item'
+import { PublicItem } from '../entity/public_item'
+import { redisDataSource } from '../redis_data_source'
 import { searchLibraryItems } from '../services/library_item'
 import { findUnseenPublicItems } from '../services/public_item'
 import { logger } from '../utils/logger'
@@ -6,9 +9,48 @@ interface JustReadFeedUpdateData {
   userId: string
 }
 
-const selectCandidates = async (userId: string) => {
+interface Candidate {
+  id: string
+  title: string
+  url: string
+  thumbnail?: string
+  previewContent?: string
+  languageCode?: string
+  author?: string
+  dir?: string
+  publishedAt?: Date
+  subscription?: string
+}
+
+const libraryItemToCandidate = (item: LibraryItem): Candidate => ({
+  id: item.id,
+  title: item.title,
+  url: item.originalUrl,
+  thumbnail: item.thumbnail || undefined,
+  previewContent: item.description || undefined,
+  languageCode: item.itemLanguage || undefined, // TODO: map to language code
+  author: item.author || undefined,
+  dir: item.directionality || undefined,
+  publishedAt: item.publishedAt || undefined,
+  subscription: item.subscription || undefined,
+})
+
+const publicItemToCandidate = (item: PublicItem): Candidate => ({
+  id: item.id,
+  title: item.title,
+  url: item.url,
+  thumbnail: item.thumbnail,
+  previewContent: item.previewContent,
+  languageCode: item.languageCode,
+  author: item.author,
+  dir: item.dir,
+  publishedAt: item.publishedAt,
+  subscription: item.source_name,
+})
+
+const selectCandidates = async (userId: string): Promise<Array<Candidate>> => {
   // get last 100 library items saved and not seen by user
-  const privateCandidates = await searchLibraryItems(
+  const libraryItems = await searchLibraryItems(
     {
       size: 100,
       includeContent: false,
@@ -17,13 +59,65 @@ const selectCandidates = async (userId: string) => {
     userId
   )
 
+  // map library items to candidates
+  const privateCandidates: Array<Candidate> = libraryItems.map(
+    libraryItemToCandidate
+  )
+
   // get candidates from public inventory
-  const publicCandidates = await findUnseenPublicItems(userId, {
+  const publicItems = await findUnseenPublicItems(userId, {
     limit: 100,
   })
 
-  // TODO: mix candidates
-  return privateCandidates.concat(publicCandidates)
+  // map public items to candidates
+  const publicCandidates: Array<Candidate> = publicItems.map(
+    publicItemToCandidate
+  )
+
+  // combine candidates
+  return [...privateCandidates, ...publicCandidates]
+}
+
+const rankCandidates = async (
+  candidates: Array<Candidate>
+): Promise<Array<Candidate>> => {
+  if (candidates.length <= 10) {
+    return candidates
+  }
+
+  // TODO: rank candidates
+  const API_URL = 'https://rank.omnivore.app'
+
+  const response = await fetch(API_URL, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ candidates }),
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to rank candidates: ${response.statusText}`)
+  }
+
+  return response.json() as Promise<Array<Candidate>>
+}
+
+const prependCandidatesToFeed = async (
+  candidates: Array<Candidate>,
+  userId: string
+) => {
+  const redisKey = `just-read-feed:${userId}`
+  const redisClient = redisDataSource.redisClient
+  if (!redisClient) {
+    throw new Error('Redis client not available')
+  }
+
+  const pipeline = redisClient.pipeline()
+  candidates.forEach((candidate) =>
+    pipeline.lpush(redisKey, JSON.stringify(candidate))
+  )
+  await pipeline.exec()
 }
 
 const updateJustReadFeed = async (data: JustReadFeedUpdateData) => {
@@ -34,7 +128,8 @@ const updateJustReadFeed = async (data: JustReadFeedUpdateData) => {
   logger.info(`Found ${candidates.length} candidates`)
 
   // TODO: integrity check on candidates?
-  // TODO: rank candidates
 
-  // TODO: prepend candidates to feed in redis
+  const rankedCandidates = await rankCandidates(candidates)
+
+  await prependCandidatesToFeed(rankedCandidates, userId)
 }
diff --git a/packages/api/src/services/public_item.ts b/packages/api/src/services/public_item.ts
index 58243abda..c34fc7151 100644
--- a/packages/api/src/services/public_item.ts
+++ b/packages/api/src/services/public_item.ts
@@ -1,4 +1,3 @@
-import { IsNull } from 'typeorm'
 import { PublicItem } from '../entity/public_item'
 import { getRepository } from '../repository'
 
@@ -8,21 +7,17 @@ export const findUnseenPublicItems = async (
     limit?: number
     offset?: number
   }
-) =>
-  getRepository(PublicItem).find({
-    where: {
-      interaction: IsNull(),
-      interaction: {
-        user: {
-          id: userId,
-        },
-        seenAt: IsNull(),
-      },
-      approved: true,
-    },
-    order: {
-      createdAt: 'DESC',
-    },
-    take: options.limit,
-    skip: options.offset,
-  })
+): Promise<Array<PublicItem>> =>
+  getRepository(PublicItem)
+    .createQueryBuilder('public_item')
+    .leftJoin(
+      'omnivore.public_item_interactions',
+      'interaction',
+      'interaction.public_item_id = public_item.id'
+    )
+    .where('interaction.user_id = :userId', { userId })
+    .andWhere('interaction.seen_at IS NULL')
+    .orderBy('public_item.created_at', 'DESC')
+    .limit(options.limit)
+    .offset(options.offset)
+    .getMany()
diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index c657dced7..597fec5fb 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -21,7 +21,8 @@ CREATE TRIGGER update_public_item_source_modtime BEFORE UPDATE ON omnivore.publi
 
 CREATE TABLE omnivore.public_item (
 	id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
-    source_id uuid NOT NULL, -- user_id or public_item_source_id
+    source_name TEXT NOT NULL,
+    source_icon TEXT,
     type TEXT NOT NULL, -- public feeds, newsletters, or user recommended
     title TEXT NOT NULL,
     url TEXT NOT NULL,
@@ -60,7 +61,7 @@ CREATE TABLE omnivore.public_item_interactions (
     saved_at TIMESTAMPTZ,
     liked_at TIMESTAMPTZ,
     broadcasted_at TIMESTAMPTZ,
-    seen_at TIMESTAMPTZ,
+    seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
     created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
     updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
 );

From cb4fc23507abeb1689245d4bd592af3a2d8dcce8 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Thu, 23 May 2024 22:07:19 +0800
Subject: [PATCH 06/33] implement justReadFeed API

---
 packages/api/src/entity/library_item.ts       |   3 +
 packages/api/src/entity/public_item.ts        |   3 +
 packages/api/src/generated/graphql.ts         |   4 +-
 packages/api/src/generated/schema.graphql     |   3 +-
 .../api/src/jobs/update_just_read_feed.ts     | 114 +++++++++++++-----
 .../api/src/resolvers/function_resolvers.ts   |   4 +-
 .../api/src/resolvers/just_read_feed/index.ts |  31 +++++
 packages/api/src/schema.ts                    |   8 +-
 .../{public_item.ts => just_read_feed.ts}     |   0
 packages/api/src/utils/createTask.ts          |  22 ++++
 .../db/migrations/0177.do.public_item.sql     |   5 +-
 .../db/migrations/0177.undo.public_item.sql   |   4 +-
 12 files changed, 163 insertions(+), 38 deletions(-)
 create mode 100644 packages/api/src/resolvers/just_read_feed/index.ts
 rename packages/api/src/services/{public_item.ts => just_read_feed.ts} (100%)

diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts
index f6a2f83de..0e9d7c259 100644
--- a/packages/api/src/entity/library_item.ts
+++ b/packages/api/src/entity/library_item.ts
@@ -207,4 +207,7 @@ export class LibraryItem {
 
   @Column('timestamptz')
   seenAt?: Date
+
+  @Column('text')
+  topic!: string
 }
diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts
index 00554c3a8..30e318a3a 100644
--- a/packages/api/src/entity/public_item.ts
+++ b/packages/api/src/entity/public_item.ts
@@ -52,4 +52,7 @@ export class PublicItem {
 
   @UpdateDateColumn()
   updatedAt!: Date
+
+  @Column('text')
+  topic!: string
 }
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 2eaa03ac2..55db3b4a0 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1392,7 +1392,6 @@ export type JustReadFeedItem = {
   thumbnail?: Maybe<Scalars['String']>;
   title: Scalars['String'];
   topic: Scalars['String'];
-  type: Scalars['String'];
   updatedAt?: Maybe<Scalars['Date']>;
   url: Scalars['String'];
 };
@@ -2272,6 +2271,8 @@ export type QueryIntegrationArgs = {
 
 
 export type QueryJustReadFeedArgs = {
+  after?: InputMaybe<Scalars['String']>;
+  first?: InputMaybe<Scalars['Int']>;
   language?: InputMaybe<Scalars['String']>;
   location?: InputMaybe<Scalars['String']>;
 };
@@ -6089,7 +6090,6 @@ export type JustReadFeedItemResolvers<ContextType = ResolverContext, ParentType
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   topic?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  type?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   updatedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
   url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 696fb8b3e..5dc0b0dc9 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1249,7 +1249,6 @@ type JustReadFeedItem {
   thumbnail: String
   title: String!
   topic: String!
-  type: String!
   updatedAt: Date
   url: String!
 }
@@ -1693,7 +1692,7 @@ type Query {
   hello: String
   integration(name: String!): IntegrationResult!
   integrations: IntegrationsResult!
-  justReadFeed(language: String, location: String): JustReadFeedResult!
+  justReadFeed(after: String, first: Int, language: String, location: String): JustReadFeedResult!
   labels: LabelsResult!
   me: User
   newsletterEmails: NewsletterEmailsResult!
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 413dd961c..33ff23d65 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -1,18 +1,21 @@
 import { LibraryItem } from '../entity/library_item'
 import { PublicItem } from '../entity/public_item'
 import { redisDataSource } from '../redis_data_source'
+import { findUnseenPublicItems } from '../services/just_read_feed'
 import { searchLibraryItems } from '../services/library_item'
-import { findUnseenPublicItems } from '../services/public_item'
 import { logger } from '../utils/logger'
 
-interface JustReadFeedUpdateData {
+export const UPDATE_JUST_READ_FEED_JOB = 'UPDATE_JUST_READ_FEED_JOB'
+
+export interface UpdateJustReadFeedJobData {
   userId: string
 }
 
-interface Candidate {
+interface JustReadFeedItem {
   id: string
   title: string
   url: string
+  topic: string
   thumbnail?: string
   previewContent?: string
   languageCode?: string
@@ -22,7 +25,17 @@ interface Candidate {
   subscription?: string
 }
 
-const libraryItemToCandidate = (item: LibraryItem): Candidate => ({
+interface JustReadFeedTopic {
+  name: string
+  items: Array<JustReadFeedItem>
+  thumbnail: string
+}
+
+interface JustReadFeed {
+  topics: Array<JustReadFeedTopic>
+}
+
+const libraryItemToFeedItem = (item: LibraryItem): JustReadFeedItem => ({
   id: item.id,
   title: item.title,
   url: item.originalUrl,
@@ -33,9 +46,10 @@ const libraryItemToCandidate = (item: LibraryItem): Candidate => ({
   dir: item.directionality || undefined,
   publishedAt: item.publishedAt || undefined,
   subscription: item.subscription || undefined,
+  topic: item.topic,
 })
 
-const publicItemToCandidate = (item: PublicItem): Candidate => ({
+const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
   id: item.id,
   title: item.title,
   url: item.url,
@@ -46,9 +60,12 @@ const publicItemToCandidate = (item: PublicItem): Candidate => ({
   dir: item.dir,
   publishedAt: item.publishedAt,
   subscription: item.source_name,
+  topic: item.topic,
 })
 
-const selectCandidates = async (userId: string): Promise<Array<Candidate>> => {
+const selectCandidates = async (
+  userId: string
+): Promise<Array<JustReadFeedItem>> => {
   // get last 100 library items saved and not seen by user
   const libraryItems = await searchLibraryItems(
     {
@@ -60,8 +77,8 @@ const selectCandidates = async (userId: string): Promise<Array<Candidate>> => {
   )
 
   // map library items to candidates
-  const privateCandidates: Array<Candidate> = libraryItems.map(
-    libraryItemToCandidate
+  const privateCandidates: Array<JustReadFeedItem> = libraryItems.map(
+    libraryItemToFeedItem
   )
 
   // get candidates from public inventory
@@ -70,19 +87,18 @@ const selectCandidates = async (userId: string): Promise<Array<Candidate>> => {
   })
 
   // map public items to candidates
-  const publicCandidates: Array<Candidate> = publicItems.map(
-    publicItemToCandidate
-  )
+  const publicCandidates: Array<JustReadFeedItem> =
+    publicItems.map(publicItemToFeedItem)
 
   // combine candidates
   return [...privateCandidates, ...publicCandidates]
 }
 
-const rankCandidates = async (
-  candidates: Array<Candidate>
-): Promise<Array<Candidate>> => {
-  if (candidates.length <= 10) {
-    return candidates
+const rankFeedItems = async (
+  feedItems: Array<JustReadFeedItem>
+): Promise<Array<JustReadFeedItem>> => {
+  if (feedItems.length <= 10) {
+    return feedItems
   }
 
   // TODO: rank candidates
@@ -93,43 +109,83 @@ const rankCandidates = async (
     headers: {
       'Content-Type': 'application/json',
     },
-    body: JSON.stringify({ candidates }),
+    body: JSON.stringify({ candidates: feedItems }),
   })
 
   if (!response.ok) {
     throw new Error(`Failed to rank candidates: ${response.statusText}`)
   }
 
-  return response.json() as Promise<Array<Candidate>>
+  return response.json() as Promise<Array<JustReadFeedItem>>
 }
 
-const prependCandidatesToFeed = async (
-  candidates: Array<Candidate>,
-  userId: string
-) => {
-  const redisKey = `just-read-feed:${userId}`
+const redisKey = (userId: string) => `just-read-feed:${userId}`
+
+export const getJustReadFeed = async (
+  userId: string,
+  limit: number,
+  offset: number
+): Promise<JustReadFeed> => {
   const redisClient = redisDataSource.redisClient
   if (!redisClient) {
     throw new Error('Redis client not available')
   }
 
+  const key = redisKey(userId)
+
+  const results = await redisClient.lrange(key, offset, offset + limit - 1)
+
+  const feedItems = results.map((item) => JSON.parse(item) as JustReadFeedItem)
+
+  const topics: Array<JustReadFeedTopic> = []
+
+  feedItems.forEach((item) => {
+    const topic = topics.find((topic) => topic.name === item.topic)
+    if (topic) {
+      topic.items.push(item)
+    } else {
+      topics.push({
+        name: item.topic,
+        thumbnail: item.thumbnail || '',
+        items: [item],
+      })
+    }
+  })
+
+  return { topics }
+}
+
+const prependItemsToFeed = async (
+  candidates: Array<JustReadFeedItem>,
+  userId: string
+) => {
+  const redisClient = redisDataSource.redisClient
+  if (!redisClient) {
+    throw new Error('Redis client not available')
+  }
+
+  const key = redisKey(userId)
+
   const pipeline = redisClient.pipeline()
   candidates.forEach((candidate) =>
-    pipeline.lpush(redisKey, JSON.stringify(candidate))
+    pipeline.lpush(key, JSON.stringify(candidate))
   )
+  // keep only the first 100 items
+  pipeline.ltrim(key, 0, 99)
+
   await pipeline.exec()
 }
 
-const updateJustReadFeed = async (data: JustReadFeedUpdateData) => {
+const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
   const { userId } = data
   logger.info(`Updating just read feed for user ${userId}`)
 
-  const candidates = await selectCandidates(userId)
-  logger.info(`Found ${candidates.length} candidates`)
+  const feedItems = await selectCandidates(userId)
+  logger.info(`Found ${feedItems.length} candidates`)
 
   // TODO: integrity check on candidates?
 
-  const rankedCandidates = await rankCandidates(candidates)
+  const rankedFeedItems = await rankFeedItems(feedItems)
 
-  await prependCandidatesToFeed(rankedCandidates, userId)
+  await prependItemsToFeed(rankedFeedItems, userId)
 }
diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts
index aa41b724d..91f8b942f 100644
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@@ -33,7 +33,6 @@ import {
   wordsCount,
 } from '../utils/helpers'
 import { createImageProxyUrl } from '../utils/imageproxy'
-import { logger } from '../utils/logger'
 import { contentConverter } from '../utils/parser'
 import {
   generateDownloadSignedUrl,
@@ -153,6 +152,7 @@ import {
   webhookResolver,
   webhooksResolver,
 } from './index'
+import { justReadFeedResolver } from './just_read_feed'
 import {
   markEmailAsItemResolver,
   recentEmailsResolver,
@@ -360,6 +360,7 @@ export const functionResolvers = {
     feeds: feedsResolver,
     scanFeeds: scanFeedsResolver,
     integration: integrationResolver,
+    justReadFeed: justReadFeedResolver,
   },
   User: {
     async intercomHash(
@@ -722,4 +723,5 @@ export const functionResolvers = {
   ...resultResolveTypeResolver('Integration'),
   ...resultResolveTypeResolver('ExportToIntegration'),
   ...resultResolveTypeResolver('ReplyToEmail'),
+  ...resultResolveTypeResolver('JustReadFeed'),
 }
diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/just_read_feed/index.ts
new file mode 100644
index 000000000..92e78a417
--- /dev/null
+++ b/packages/api/src/resolvers/just_read_feed/index.ts
@@ -0,0 +1,31 @@
+import {
+  JustReadFeedError,
+  JustReadFeedSuccess,
+  QueryJustReadFeedArgs,
+} from '../../generated/graphql'
+import { getJustReadFeed } from '../../jobs/update_just_read_feed'
+import { enqueueUpdateJustReadFeed } from '../../utils/createTask'
+import { authorized } from '../../utils/gql-utils'
+
+export const justReadFeedResolver = authorized<
+  JustReadFeedSuccess,
+  JustReadFeedError,
+  QueryJustReadFeedArgs
+>(async (_, { first, after }, { uid, log }) => {
+  first = first || 10
+  after = after || '0'
+  const offset = parseInt(after, 10)
+
+  const feed = await getJustReadFeed(uid, first, offset)
+  if (feed.topics.length === 0) {
+    log.info('No feed items found, updating feed')
+
+    await enqueueUpdateJustReadFeed({ userId: uid })
+
+    return {
+      topics: [],
+    }
+  }
+
+  return feed
+})
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index c6dd0ed50..9eabe1a73 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3128,7 +3128,6 @@ const schema = gql`
     updatedAt: Date
     comments: [String!]
     author: String
-    type: String!
     languageCode: String
     dir: String
     seen_at: Date
@@ -3350,7 +3349,12 @@ const schema = gql`
     feeds(input: FeedsInput!): FeedsResult!
     discoverFeeds: DiscoverFeedResult!
     scanFeeds(input: ScanFeedsInput!): ScanFeedsResult!
-    justReadFeed(location: String, language: String): JustReadFeedResult!
+    justReadFeed(
+      location: String
+      language: String
+      first: Int
+      after: String
+    ): JustReadFeedResult!
   }
 `
 
diff --git a/packages/api/src/services/public_item.ts b/packages/api/src/services/just_read_feed.ts
similarity index 100%
rename from packages/api/src/services/public_item.ts
rename to packages/api/src/services/just_read_feed.ts
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index e06c1bb99..5903d8193 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -53,6 +53,10 @@ import {
   UPDATE_HIGHLIGHT_JOB,
   UPDATE_LABELS_JOB,
 } from '../jobs/update_db'
+import {
+  UpdateJustReadFeedJobData,
+  UPDATE_JUST_READ_FEED_JOB,
+} from '../jobs/update_just_read_feed'
 import {
   UploadContentJobData,
   UPLOAD_CONTENT_JOB,
@@ -85,6 +89,7 @@ export const getJobPriority = (jobName: string): number => {
     case UPDATE_HIGHLIGHT_JOB:
     case SYNC_READ_POSITIONS_JOB_NAME:
     case SEND_EMAIL_JOB:
+    case UPDATE_JUST_READ_FEED_JOB:
       return 1
     case TRIGGER_RULE_JOB_NAME:
     case CALL_WEBHOOK_JOB_NAME:
@@ -981,4 +986,21 @@ export const enqueueBulkUploadContentJob = async (
   return queue.addBulk(jobs)
 }
 
+export const enqueueUpdateJustReadFeed = async (
+  data: UpdateJustReadFeedJobData
+) => {
+  const queue = await getBackendQueue()
+  if (!queue) {
+    return undefined
+  }
+
+  return queue.add(UPDATE_JUST_READ_FEED_JOB, data, {
+    jobId: `${UPDATE_JUST_READ_FEED_JOB}_${data.userId}_${JOB_VERSION}`,
+    removeOnComplete: true,
+    removeOnFail: true,
+    priority: getJobPriority(UPDATE_JUST_READ_FEED_JOB),
+    attempts: 3,
+  })
+}
+
 export default createHttpTaskWithToken
diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index 597fec5fb..d5c3b31d4 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -26,6 +26,7 @@ CREATE TABLE omnivore.public_item (
     type TEXT NOT NULL, -- public feeds, newsletters, or user recommended
     title TEXT NOT NULL,
     url TEXT NOT NULL,
+    topic TEXT NOT NULL,
     approved BOOLEAN NOT NULL DEFAULT FALSE,
     thumbnail TEXT,
     preview_content TEXT,
@@ -71,6 +72,8 @@ CREATE INDEX public_item_interaction_public_item_id_idx ON omnivore.public_item_
 CREATE TRIGGER update_public_item_interactions_modtime BEFORE UPDATE ON omnivore.public_item_interactions FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
 
 
-ALTER TABLE omnivore.library_item ADD COLUMN seen_at timestamptz;
+ALTER TABLE omnivore.library_item 
+    ADD COLUMN seen_at timestamptz,
+    ADD COLUMN topic TEXT NOT NULL;
 
 COMMIT;
diff --git a/packages/db/migrations/0177.undo.public_item.sql b/packages/db/migrations/0177.undo.public_item.sql
index 36c5f8165..5b9704f4c 100755
--- a/packages/db/migrations/0177.undo.public_item.sql
+++ b/packages/db/migrations/0177.undo.public_item.sql
@@ -9,6 +9,8 @@ DROP TABLE omnivore.public_item_stats;
 DROP TABLE omnivore.public_item;
 DROP TABLE omnivore.public_item_source;
 
-ALTER TABLE omnivore.library_item DROP COLUMN seen_at;
+ALTER TABLE omnivore.library_item 
+    DROP COLUMN seen_at,
+    DROP COLUMN topic;
 
 COMMIT;

From a48a61467670211aae2089e18468ddbe067378f1 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 24 May 2024 14:52:21 +0800
Subject: [PATCH 07/33] save just read feed in redis sorted set

---
 packages/api/src/entity/library_item.ts       |   5 +-
 packages/api/src/entity/public_item.ts        |  12 +-
 .../api/src/entity/public_item_interaction.ts |   3 +
 packages/api/src/generated/graphql.ts         |  81 +++++++-------
 packages/api/src/generated/schema.graphql     |  16 ++-
 .../api/src/jobs/update_just_read_feed.ts     | 104 +++++++++++++-----
 packages/api/src/queue-processor.ts           |   6 +
 .../api/src/resolvers/just_read_feed/index.ts |  28 +----
 packages/api/src/schema.ts                    |  21 ++--
 .../db/migrations/0177.do.public_item.sql     |  20 ++--
 .../db/migrations/0177.undo.public_item.sql   |   1 +
 11 files changed, 184 insertions(+), 113 deletions(-)

diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts
index 0e9d7c259..2950ff9a6 100644
--- a/packages/api/src/entity/library_item.ts
+++ b/packages/api/src/entity/library_item.ts
@@ -209,5 +209,8 @@ export class LibraryItem {
   seenAt?: Date
 
   @Column('text')
-  topic!: string
+  topic?: string
+
+  @Column('timestamptz')
+  digestedAt?: Date
 }
diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts
index 30e318a3a..df82e1713 100644
--- a/packages/api/src/entity/public_item.ts
+++ b/packages/api/src/entity/public_item.ts
@@ -12,10 +12,10 @@ export class PublicItem {
   id!: string
 
   @Column('text')
-  source_name!: string
+  sourceName!: string
 
   @Column('text')
-  source_icon!: string
+  sourceIcon?: string
 
   @Column('text')
   type!: string
@@ -54,5 +54,11 @@ export class PublicItem {
   updatedAt!: Date
 
   @Column('text')
-  topic!: string
+  topic?: string
+
+  @Column('integer')
+  wordCount?: number
+
+  @Column('text')
+  siteName?: string
 }
diff --git a/packages/api/src/entity/public_item_interaction.ts b/packages/api/src/entity/public_item_interaction.ts
index efb370d58..dc192e72a 100644
--- a/packages/api/src/entity/public_item_interaction.ts
+++ b/packages/api/src/entity/public_item_interaction.ts
@@ -41,4 +41,7 @@ export class PublicItemInteraction {
 
   @Column('timestamptz')
   updated!: Date
+
+  @Column('timestamptz')
+  digested?: Date
 }
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 55db3b4a0..a3d09f2eb 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1391,9 +1391,10 @@ export type JustReadFeedItem = {
   source?: Maybe<JustReadFeedSource>;
   thumbnail?: Maybe<Scalars['String']>;
   title: Scalars['String'];
-  topic: Scalars['String'];
+  topic?: Maybe<Scalars['String']>;
   updatedAt?: Maybe<Scalars['Date']>;
   url: Scalars['String'];
+  wordCount?: Maybe<Scalars['Int']>;
 };
 
 export type JustReadFeedResult = JustReadFeedError | JustReadFeedSuccess;
@@ -1416,7 +1417,7 @@ export type JustReadFeedSuccess = {
 export type JustReadFeedTopic = {
   __typename?: 'JustReadFeedTopic';
   items: Array<JustReadFeedItem>;
-  name: Scalars['String'];
+  name?: Maybe<Scalars['String']>;
   thumbnail?: Maybe<Scalars['String']>;
 };
 
@@ -2072,6 +2073,11 @@ export type MutationUploadImportFileArgs = {
   type: UploadImportFileType;
 };
 
+export type MySubscriptionRootType = {
+  __typename?: 'MySubscriptionRootType';
+  justReadFeed: JustReadFeedResult;
+};
+
 export type NewsletterEmail = {
   __typename?: 'NewsletterEmail';
   address: Scalars['String'];
@@ -2218,7 +2224,6 @@ export type Query = {
   hello?: Maybe<Scalars['String']>;
   integration: IntegrationResult;
   integrations: IntegrationsResult;
-  justReadFeed: JustReadFeedResult;
   labels: LabelsResult;
   me?: Maybe<User>;
   newsletterEmails: NewsletterEmailsResult;
@@ -2270,14 +2275,6 @@ export type QueryIntegrationArgs = {
 };
 
 
-export type QueryJustReadFeedArgs = {
-  after?: InputMaybe<Scalars['String']>;
-  first?: InputMaybe<Scalars['Int']>;
-  language?: InputMaybe<Scalars['String']>;
-  location?: InputMaybe<Scalars['String']>;
-};
-
-
 export type QueryRulesArgs = {
   enabled?: InputMaybe<Scalars['Boolean']>;
 };
@@ -4317,6 +4314,7 @@ export type ResolversTypes = {
   MoveToFolderResult: ResolversTypes['MoveToFolderError'] | ResolversTypes['MoveToFolderSuccess'];
   MoveToFolderSuccess: ResolverTypeWrapper<MoveToFolderSuccess>;
   Mutation: ResolverTypeWrapper<{}>;
+  MySubscriptionRootType: ResolverTypeWrapper<{}>;
   NewsletterEmail: ResolverTypeWrapper<NewsletterEmail>;
   NewsletterEmailsError: ResolverTypeWrapper<NewsletterEmailsError>;
   NewsletterEmailsErrorCode: NewsletterEmailsErrorCode;
@@ -4494,7 +4492,7 @@ export type ResolversTypes = {
   SubscribeInput: SubscribeInput;
   SubscribeResult: ResolversTypes['SubscribeError'] | ResolversTypes['SubscribeSuccess'];
   SubscribeSuccess: ResolverTypeWrapper<SubscribeSuccess>;
-  Subscription: ResolverTypeWrapper<{}>;
+  Subscription: ResolverTypeWrapper<Subscription>;
   SubscriptionStatus: SubscriptionStatus;
   SubscriptionType: SubscriptionType;
   SubscriptionsError: ResolverTypeWrapper<SubscriptionsError>;
@@ -4856,6 +4854,7 @@ export type ResolversParentTypes = {
   MoveToFolderResult: ResolversParentTypes['MoveToFolderError'] | ResolversParentTypes['MoveToFolderSuccess'];
   MoveToFolderSuccess: MoveToFolderSuccess;
   Mutation: {};
+  MySubscriptionRootType: {};
   NewsletterEmail: NewsletterEmail;
   NewsletterEmailsError: NewsletterEmailsError;
   NewsletterEmailsResult: ResolversParentTypes['NewsletterEmailsError'] | ResolversParentTypes['NewsletterEmailsSuccess'];
@@ -4994,7 +4993,7 @@ export type ResolversParentTypes = {
   SubscribeInput: SubscribeInput;
   SubscribeResult: ResolversParentTypes['SubscribeError'] | ResolversParentTypes['SubscribeSuccess'];
   SubscribeSuccess: SubscribeSuccess;
-  Subscription: {};
+  Subscription: Subscription;
   SubscriptionsError: SubscriptionsError;
   SubscriptionsResult: ResolversParentTypes['SubscriptionsError'] | ResolversParentTypes['SubscriptionsSuccess'];
   SubscriptionsSuccess: SubscriptionsSuccess;
@@ -6089,9 +6088,10 @@ export type JustReadFeedItemResolvers<ContextType = ResolverContext, ParentType
   source?: Resolver<Maybe<ResolversTypes['JustReadFeedSource']>, ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  topic?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  topic?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   updatedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
   url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  wordCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
@@ -6116,7 +6116,7 @@ export type JustReadFeedSuccessResolvers<ContextType = ResolverContext, ParentTy
 
 export type JustReadFeedTopicResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedTopic'] = ResolversParentTypes['JustReadFeedTopic']> = {
   items?: Resolver<Array<ResolversTypes['JustReadFeedItem']>, ParentType, ContextType>;
-  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  name?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
@@ -6357,6 +6357,10 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
   uploadImportFile?: Resolver<ResolversTypes['UploadImportFileResult'], ParentType, ContextType, RequireFields<MutationUploadImportFileArgs, 'contentType' | 'type'>>;
 };
 
+export type MySubscriptionRootTypeResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['MySubscriptionRootType'] = ResolversParentTypes['MySubscriptionRootType']> = {
+  justReadFeed?: SubscriptionResolver<ResolversTypes['JustReadFeedResult'], "justReadFeed", ParentType, ContextType>;
+};
+
 export type NewsletterEmailResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['NewsletterEmail'] = ResolversParentTypes['NewsletterEmail']> = {
   address?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   confirmationCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
@@ -6448,7 +6452,6 @@ export type QueryResolvers<ContextType = ResolverContext, ParentType extends Res
   hello?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   integration?: Resolver<ResolversTypes['IntegrationResult'], ParentType, ContextType, RequireFields<QueryIntegrationArgs, 'name'>>;
   integrations?: Resolver<ResolversTypes['IntegrationsResult'], ParentType, ContextType>;
-  justReadFeed?: Resolver<ResolversTypes['JustReadFeedResult'], ParentType, ContextType, Partial<QueryJustReadFeedArgs>>;
   labels?: Resolver<ResolversTypes['LabelsResult'], ParentType, ContextType>;
   me?: Resolver<Maybe<ResolversTypes['User']>, ParentType, ContextType>;
   newsletterEmails?: Resolver<ResolversTypes['NewsletterEmailsResult'], ParentType, ContextType>;
@@ -7032,28 +7035,29 @@ export type SubscribeSuccessResolvers<ContextType = ResolverContext, ParentType
 };
 
 export type SubscriptionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Subscription'] = ResolversParentTypes['Subscription']> = {
-  autoAddToLibrary?: SubscriptionResolver<Maybe<ResolversTypes['Boolean']>, "autoAddToLibrary", ParentType, ContextType>;
-  count?: SubscriptionResolver<ResolversTypes['Int'], "count", ParentType, ContextType>;
-  createdAt?: SubscriptionResolver<ResolversTypes['Date'], "createdAt", ParentType, ContextType>;
-  description?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "description", ParentType, ContextType>;
-  failedAt?: SubscriptionResolver<Maybe<ResolversTypes['Date']>, "failedAt", ParentType, ContextType>;
-  fetchContent?: SubscriptionResolver<ResolversTypes['Boolean'], "fetchContent", ParentType, ContextType>;
-  fetchContentType?: SubscriptionResolver<ResolversTypes['FetchContentType'], "fetchContentType", ParentType, ContextType>;
-  folder?: SubscriptionResolver<ResolversTypes['String'], "folder", ParentType, ContextType>;
-  icon?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "icon", ParentType, ContextType>;
-  id?: SubscriptionResolver<ResolversTypes['ID'], "id", ParentType, ContextType>;
-  isPrivate?: SubscriptionResolver<Maybe<ResolversTypes['Boolean']>, "isPrivate", ParentType, ContextType>;
-  lastFetchedAt?: SubscriptionResolver<Maybe<ResolversTypes['Date']>, "lastFetchedAt", ParentType, ContextType>;
-  mostRecentItemDate?: SubscriptionResolver<Maybe<ResolversTypes['Date']>, "mostRecentItemDate", ParentType, ContextType>;
-  name?: SubscriptionResolver<ResolversTypes['String'], "name", ParentType, ContextType>;
-  newsletterEmail?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "newsletterEmail", ParentType, ContextType>;
-  refreshedAt?: SubscriptionResolver<Maybe<ResolversTypes['Date']>, "refreshedAt", ParentType, ContextType>;
-  status?: SubscriptionResolver<ResolversTypes['SubscriptionStatus'], "status", ParentType, ContextType>;
-  type?: SubscriptionResolver<ResolversTypes['SubscriptionType'], "type", ParentType, ContextType>;
-  unsubscribeHttpUrl?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "unsubscribeHttpUrl", ParentType, ContextType>;
-  unsubscribeMailTo?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "unsubscribeMailTo", ParentType, ContextType>;
-  updatedAt?: SubscriptionResolver<Maybe<ResolversTypes['Date']>, "updatedAt", ParentType, ContextType>;
-  url?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "url", ParentType, ContextType>;
+  autoAddToLibrary?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  count?: Resolver<ResolversTypes['Int'], ParentType, ContextType>;
+  createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
+  description?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  failedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  fetchContent?: Resolver<ResolversTypes['Boolean'], ParentType, ContextType>;
+  fetchContentType?: Resolver<ResolversTypes['FetchContentType'], ParentType, ContextType>;
+  folder?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  icon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  isPrivate?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  lastFetchedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  mostRecentItemDate?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  newsletterEmail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  refreshedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  status?: Resolver<ResolversTypes['SubscriptionStatus'], ParentType, ContextType>;
+  type?: Resolver<ResolversTypes['SubscriptionType'], ParentType, ContextType>;
+  unsubscribeHttpUrl?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  unsubscribeMailTo?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  updatedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
 export type SubscriptionsErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['SubscriptionsError'] = ResolversParentTypes['SubscriptionsError']> = {
@@ -7676,6 +7680,7 @@ export type Resolvers<ContextType = ResolverContext> = {
   MoveToFolderResult?: MoveToFolderResultResolvers<ContextType>;
   MoveToFolderSuccess?: MoveToFolderSuccessResolvers<ContextType>;
   Mutation?: MutationResolvers<ContextType>;
+  MySubscriptionRootType?: MySubscriptionRootTypeResolvers<ContextType>;
   NewsletterEmail?: NewsletterEmailResolvers<ContextType>;
   NewsletterEmailsError?: NewsletterEmailsErrorResolvers<ContextType>;
   NewsletterEmailsResult?: NewsletterEmailsResultResolvers<ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 5dc0b0dc9..0474deea6 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1,3 +1,9 @@
+schema {
+  query: Query
+  mutation: Mutation
+  subscription: MySubscriptionRootType
+}
+
 directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION
 
 type AddDiscoverFeedError {
@@ -1248,9 +1254,10 @@ type JustReadFeedItem {
   source: JustReadFeedSource
   thumbnail: String
   title: String!
-  topic: String!
+  topic: String
   updatedAt: Date
   url: String!
+  wordCount: Int
 }
 
 union JustReadFeedResult = JustReadFeedError | JustReadFeedSuccess
@@ -1270,7 +1277,7 @@ type JustReadFeedSuccess {
 
 type JustReadFeedTopic {
   items: [JustReadFeedItem!]!
-  name: String!
+  name: String
   thumbnail: String
 }
 
@@ -1555,6 +1562,10 @@ type Mutation {
   uploadImportFile(contentType: String!, type: UploadImportFileType!): UploadImportFileResult!
 }
 
+type MySubscriptionRootType {
+  justReadFeed: JustReadFeedResult!
+}
+
 type NewsletterEmail {
   address: String!
   confirmationCode: String
@@ -1692,7 +1703,6 @@ type Query {
   hello: String
   integration(name: String!): IntegrationResult!
   integrations: IntegrationsResult!
-  justReadFeed(after: String, first: Int, language: String, location: String): JustReadFeedResult!
   labels: LabelsResult!
   me: User
   newsletterEmails: NewsletterEmailsResult!
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 33ff23d65..a72cf61de 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -1,8 +1,10 @@
 import { LibraryItem } from '../entity/library_item'
 import { PublicItem } from '../entity/public_item'
+import { User } from '../entity/user'
 import { redisDataSource } from '../redis_data_source'
 import { findUnseenPublicItems } from '../services/just_read_feed'
 import { searchLibraryItems } from '../services/library_item'
+import { findActiveUser } from '../services/user'
 import { logger } from '../utils/logger'
 
 export const UPDATE_JUST_READ_FEED_JOB = 'UPDATE_JUST_READ_FEED_JOB'
@@ -15,18 +17,22 @@ interface JustReadFeedItem {
   id: string
   title: string
   url: string
-  topic: string
+  createdAt: Date
+  sourceName: string
+
+  siteName?: string
+  topic?: string
   thumbnail?: string
   previewContent?: string
   languageCode?: string
   author?: string
   dir?: string
-  publishedAt?: Date
-  subscription?: string
+  wordCount?: number
+  sourceIcon?: string
 }
 
 interface JustReadFeedTopic {
-  name: string
+  name?: string
   items: Array<JustReadFeedItem>
   thumbnail: string
 }
@@ -35,7 +41,10 @@ interface JustReadFeed {
   topics: Array<JustReadFeedTopic>
 }
 
-const libraryItemToFeedItem = (item: LibraryItem): JustReadFeedItem => ({
+const libraryItemToFeedItem = (
+  user: User,
+  item: LibraryItem
+): JustReadFeedItem => ({
   id: item.id,
   title: item.title,
   url: item.originalUrl,
@@ -44,9 +53,12 @@ const libraryItemToFeedItem = (item: LibraryItem): JustReadFeedItem => ({
   languageCode: item.itemLanguage || undefined, // TODO: map to language code
   author: item.author || undefined,
   dir: item.directionality || undefined,
-  publishedAt: item.publishedAt || undefined,
-  subscription: item.subscription || undefined,
+  createdAt: item.createdAt,
   topic: item.topic,
+  wordCount: item.wordCount || undefined,
+  sourceIcon: user.profile.pictureUrl || undefined,
+  sourceName: user.name,
+  siteName: item.siteName || undefined,
 })
 
 const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
@@ -58,14 +70,23 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
   languageCode: item.languageCode,
   author: item.author,
   dir: item.dir,
-  publishedAt: item.publishedAt,
-  subscription: item.source_name,
+  createdAt: item.createdAt,
   topic: item.topic,
+  wordCount: item.wordCount || undefined,
+  sourceIcon: item.sourceIcon,
+  sourceName: item.sourceName,
+  siteName: item.siteName,
 })
 
+interface FeedItemScore {
+  id: string
+  score: number
+}
+
 const selectCandidates = async (
-  userId: string
+  user: User
 ): Promise<Array<JustReadFeedItem>> => {
+  const userId = user.id
   // get last 100 library items saved and not seen by user
   const libraryItems = await searchLibraryItems(
     {
@@ -78,7 +99,7 @@ const selectCandidates = async (
 
   // map library items to candidates
   const privateCandidates: Array<JustReadFeedItem> = libraryItems.map(
-    libraryItemToFeedItem
+    (libraryItem) => libraryItemToFeedItem(user, libraryItem)
   )
 
   // get candidates from public inventory
@@ -98,11 +119,12 @@ const rankFeedItems = async (
   feedItems: Array<JustReadFeedItem>
 ): Promise<Array<JustReadFeedItem>> => {
   if (feedItems.length <= 10) {
+    // no need to rank if there are less than 10 candidates
     return feedItems
   }
 
-  // TODO: rank candidates
-  const API_URL = 'https://rank.omnivore.app'
+  // TODO: get score of candidates
+  const API_URL = 'https://score.omnivore.app' // fake URL
 
   const response = await fetch(API_URL, {
     method: 'POST',
@@ -113,18 +135,25 @@ const rankFeedItems = async (
   })
 
   if (!response.ok) {
-    throw new Error(`Failed to rank candidates: ${response.statusText}`)
+    throw new Error(`Failed to score candidates: ${response.statusText}`)
   }
 
-  return response.json() as Promise<Array<JustReadFeedItem>>
+  const scores = (await response.json()) as Array<FeedItemScore>
+
+  // rank candidates by score in ascending order
+  return feedItems.sort((a, b) => {
+    const scoreA = scores.find((score) => score.id === a.id)?.score || 0
+    const scoreB = scores.find((score) => score.id === b.id)?.score || 0
+
+    return scoreA - scoreB
+  })
 }
 
 const redisKey = (userId: string) => `just-read-feed:${userId}`
+const MAX_FEED_ITEMS = 500
 
 export const getJustReadFeed = async (
-  userId: string,
-  limit: number,
-  offset: number
+  userId: string
 ): Promise<JustReadFeed> => {
   const redisClient = redisDataSource.redisClient
   if (!redisClient) {
@@ -133,7 +162,7 @@ export const getJustReadFeed = async (
 
   const key = redisKey(userId)
 
-  const results = await redisClient.lrange(key, offset, offset + limit - 1)
+  const results = await redisClient.zrevrange(key, 0, MAX_FEED_ITEMS)
 
   const feedItems = results.map((item) => JSON.parse(item) as JustReadFeedItem)
 
@@ -155,8 +184,8 @@ export const getJustReadFeed = async (
   return { topics }
 }
 
-const prependItemsToFeed = async (
-  candidates: Array<JustReadFeedItem>,
+const appendItemsToFeed = async (
+  feedItems: Array<JustReadFeedItem>,
   userId: string
 ) => {
   const redisClient = redisDataSource.redisClient
@@ -166,26 +195,43 @@ const prependItemsToFeed = async (
 
   const key = redisKey(userId)
 
+  // store candidates in redis sorted set
   const pipeline = redisClient.pipeline()
-  candidates.forEach((candidate) =>
-    pipeline.lpush(key, JSON.stringify(candidate))
-  )
-  // keep only the first 100 items
-  pipeline.ltrim(key, 0, 99)
+
+  const scoreMembers = feedItems.flatMap((item) => [
+    Date.now() + 86_400_000, // items expire in 24 hours
+    JSON.stringify(item),
+  ])
+  // add candidates to the sorted set
+  pipeline.zadd(key, ...scoreMembers)
+
+  // remove expired items and keep only the top 500
+  pipeline.zremrangebyrank(key, 0, -(MAX_FEED_ITEMS + 1))
+  pipeline.zremrangebyscore(key, '-inf', Date.now())
 
   await pipeline.exec()
 }
 
-const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
+export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
   const { userId } = data
+  const user = await findActiveUser(userId)
+  if (!user) {
+    logger.error(`User ${userId} not found`)
+    return
+  }
+
   logger.info(`Updating just read feed for user ${userId}`)
 
-  const feedItems = await selectCandidates(userId)
+  const feedItems = await selectCandidates(user)
   logger.info(`Found ${feedItems.length} candidates`)
 
   // TODO: integrity check on candidates?
 
   const rankedFeedItems = await rankFeedItems(feedItems)
 
-  await prependItemsToFeed(rankedFeedItems, userId)
+  // TODO: filtering
+  // get top 100 ranked feed items
+  const filteredFeedItems = rankedFeedItems.slice(0, 100)
+
+  await appendItemsToFeed(filteredFeedItems, userId)
 }
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index 359bdfcf1..edb634e70 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -59,6 +59,10 @@ import {
   UPDATE_HIGHLIGHT_JOB,
   UPDATE_LABELS_JOB,
 } from './jobs/update_db'
+import {
+  updateJustReadFeed,
+  UPDATE_JUST_READ_FEED_JOB,
+} from './jobs/update_just_read_feed'
 import { updatePDFContentJob } from './jobs/update_pdf_content'
 import { uploadContentJob, UPLOAD_CONTENT_JOB } from './jobs/upload_content'
 import { redisDataSource } from './redis_data_source'
@@ -185,6 +189,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           return createDigest(job.data)
         case UPLOAD_CONTENT_JOB:
           return uploadContentJob(job.data)
+        case UPDATE_JUST_READ_FEED_JOB:
+          return updateJustReadFeed(job.data)
         default:
           logger.warning(`[queue-processor] unhandled job: ${job.name}`)
       }
diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/just_read_feed/index.ts
index 92e78a417..c907d1ddb 100644
--- a/packages/api/src/resolvers/just_read_feed/index.ts
+++ b/packages/api/src/resolvers/just_read_feed/index.ts
@@ -1,31 +1,13 @@
-import {
-  JustReadFeedError,
-  JustReadFeedSuccess,
-  QueryJustReadFeedArgs,
-} from '../../generated/graphql'
+import { JustReadFeedError, JustReadFeedSuccess } from '../../generated/graphql'
 import { getJustReadFeed } from '../../jobs/update_just_read_feed'
-import { enqueueUpdateJustReadFeed } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 
 export const justReadFeedResolver = authorized<
   JustReadFeedSuccess,
-  JustReadFeedError,
-  QueryJustReadFeedArgs
->(async (_, { first, after }, { uid, log }) => {
-  first = first || 10
-  after = after || '0'
-  const offset = parseInt(after, 10)
-
-  const feed = await getJustReadFeed(uid, first, offset)
-  if (feed.topics.length === 0) {
-    log.info('No feed items found, updating feed')
-
-    await enqueueUpdateJustReadFeed({ userId: uid })
-
-    return {
-      topics: [],
-    }
-  }
+  JustReadFeedError
+>(async (_, __, { uid, log }) => {
+  const feed = await getJustReadFeed(uid)
+  log.info('Just read feed fetched')
 
   return feed
 })
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 9eabe1a73..ad6d04cbc 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3114,7 +3114,7 @@ const schema = gql`
   type JustReadFeedItem {
     id: ID!
     title: String!
-    topic: String!
+    topic: String
     url: String!
     thumbnail: String
     publishedAt: Date
@@ -3131,10 +3131,11 @@ const schema = gql`
     languageCode: String
     dir: String
     seen_at: Date
+    wordCount: Int
   }
 
   type JustReadFeedTopic {
-    name: String!
+    name: String
     items: [JustReadFeedItem!]!
     thumbnail: String
   }
@@ -3154,6 +3155,10 @@ const schema = gql`
 
   union JustReadFeedResult = JustReadFeedSuccess | JustReadFeedError
 
+  type MySubscriptionRootType {
+    justReadFeed: JustReadFeedResult!
+  }
+
   # Mutations
   type Mutation {
     googleLogin(input: GoogleLoginInput!): LoginResult!
@@ -3349,12 +3354,12 @@ const schema = gql`
     feeds(input: FeedsInput!): FeedsResult!
     discoverFeeds: DiscoverFeedResult!
     scanFeeds(input: ScanFeedsInput!): ScanFeedsResult!
-    justReadFeed(
-      location: String
-      language: String
-      first: Int
-      after: String
-    ): JustReadFeedResult!
+  }
+
+  schema {
+    query: Query
+    mutation: Mutation
+    subscription: MySubscriptionRootType
   }
 `
 
diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index d5c3b31d4..59df6453a 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -12,8 +12,8 @@ CREATE TABLE omnivore.public_item_source (
     url TEXT NOT NULL,
     language_codes TEXT[] NOT NULL,
     approved BOOLEAN NOT NULL DEFAULT FALSE,
-    created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
 CREATE TRIGGER update_public_item_source_modtime BEFORE UPDATE ON omnivore.public_item_source FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
@@ -26,7 +26,7 @@ CREATE TABLE omnivore.public_item (
     type TEXT NOT NULL, -- public feeds, newsletters, or user recommended
     title TEXT NOT NULL,
     url TEXT NOT NULL,
-    topic TEXT NOT NULL,
+    topic TEXT,
     approved BOOLEAN NOT NULL DEFAULT FALSE,
     thumbnail TEXT,
     preview_content TEXT,
@@ -34,8 +34,10 @@ CREATE TABLE omnivore.public_item (
     author TEXT,
     dir TEXT,
     published_at timestamptz,
-    created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
+    word_count INT,
+    site_name TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
 CREATE TRIGGER update_public_item_modtime BEFORE UPDATE ON omnivore.public_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
@@ -47,8 +49,8 @@ CREATE TABLE omnivore.public_item_stats (
     save_count INT NOT NULL DEFAULT 0,
     like_count INT NOT NULL DEFAULT 0,
     broadcast_count INT NOT NULL DEFAULT 0,
-    created_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updated_at timestamptz NOT NULL DEFAULT CURRENT_TIMESTAMP
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
 CREATE INDEX public_item_stats_public_item_id_idx ON omnivore.public_item_stats(public_item_id);
@@ -63,6 +65,7 @@ CREATE TABLE omnivore.public_item_interactions (
     liked_at TIMESTAMPTZ,
     broadcasted_at TIMESTAMPTZ,
     seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    digested_at TIMESTAMPTZ,
     created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
     updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
@@ -74,6 +77,7 @@ CREATE TRIGGER update_public_item_interactions_modtime BEFORE UPDATE ON omnivore
 
 ALTER TABLE omnivore.library_item 
     ADD COLUMN seen_at timestamptz,
-    ADD COLUMN topic TEXT NOT NULL;
+    ADD COLUMN digested_at timestamptz,
+    ADD COLUMN topic TEXT;
 
 COMMIT;
diff --git a/packages/db/migrations/0177.undo.public_item.sql b/packages/db/migrations/0177.undo.public_item.sql
index 5b9704f4c..a58f38333 100755
--- a/packages/db/migrations/0177.undo.public_item.sql
+++ b/packages/db/migrations/0177.undo.public_item.sql
@@ -11,6 +11,7 @@ DROP TABLE omnivore.public_item_source;
 
 ALTER TABLE omnivore.library_item 
     DROP COLUMN seen_at,
+    DROP COLUMN digested_at,
     DROP COLUMN topic;
 
 COMMIT;

From f2d2e852e1f02def9374cc0e548ad568e44c13bf Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 24 May 2024 15:32:14 +0800
Subject: [PATCH 08/33] grant db permission

---
 packages/api/src/entity/public_item.ts        |  2 +-
 packages/api/src/generated/graphql.ts         | 28 +++++-----
 packages/api/src/generated/schema.graphql     | 14 ++---
 .../api/src/jobs/update_just_read_feed.ts     | 53 ++++++++++++++-----
 .../api/src/resolvers/just_read_feed/index.ts |  7 +++
 packages/api/src/schema.ts                    | 13 ++---
 packages/api/src/services/just_read_feed.ts   | 42 +++++++++------
 .../db/migrations/0177.do.public_item.sql     |  6 ++-
 8 files changed, 111 insertions(+), 54 deletions(-)

diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts
index df82e1713..0bf5c7862 100644
--- a/packages/api/src/entity/public_item.ts
+++ b/packages/api/src/entity/public_item.ts
@@ -6,7 +6,7 @@ import {
   UpdateDateColumn,
 } from 'typeorm'
 
-@Entity()
+@Entity({ name: 'public_item' })
 export class PublicItem {
   @PrimaryGeneratedColumn('uuid')
   id!: string
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index a3d09f2eb..514a7d2ba 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1378,21 +1378,22 @@ export type JustReadFeedItem = {
   author?: Maybe<Scalars['String']>;
   broadcastCount?: Maybe<Scalars['Int']>;
   comments?: Maybe<Array<Scalars['String']>>;
-  createdAt?: Maybe<Scalars['Date']>;
+  createdAt: Scalars['Date'];
   dir?: Maybe<Scalars['String']>;
   highlights?: Maybe<Scalars['String']>;
   id: Scalars['ID'];
   languageCode?: Maybe<Scalars['String']>;
   likedCount?: Maybe<Scalars['Int']>;
   previewContent?: Maybe<Scalars['String']>;
-  publishedAt?: Maybe<Scalars['Date']>;
   savedCount?: Maybe<Scalars['Int']>;
   seen_at?: Maybe<Scalars['Date']>;
-  source?: Maybe<JustReadFeedSource>;
+  siteName?: Maybe<Scalars['String']>;
+  sourceIcon?: Maybe<Scalars['String']>;
+  sourceName: Scalars['String'];
   thumbnail?: Maybe<Scalars['String']>;
   title: Scalars['String'];
   topic?: Maybe<Scalars['String']>;
-  updatedAt?: Maybe<Scalars['Date']>;
+  updatedAt: Scalars['Date'];
   url: Scalars['String'];
   wordCount?: Maybe<Scalars['Int']>;
 };
@@ -1406,7 +1407,7 @@ export type JustReadFeedSource = {
   name: Scalars['String'];
   thumbnail?: Maybe<Scalars['String']>;
   topics?: Maybe<Array<Scalars['String']>>;
-  url?: Maybe<Scalars['String']>;
+  url: Scalars['String'];
 };
 
 export type JustReadFeedSuccess = {
@@ -2075,7 +2076,7 @@ export type MutationUploadImportFileArgs = {
 
 export type MySubscriptionRootType = {
   __typename?: 'MySubscriptionRootType';
-  justReadFeed: JustReadFeedResult;
+  hello?: Maybe<Scalars['String']>;
 };
 
 export type NewsletterEmail = {
@@ -2224,6 +2225,7 @@ export type Query = {
   hello?: Maybe<Scalars['String']>;
   integration: IntegrationResult;
   integrations: IntegrationsResult;
+  justReadFeed: JustReadFeedResult;
   labels: LabelsResult;
   me?: Maybe<User>;
   newsletterEmails: NewsletterEmailsResult;
@@ -6075,21 +6077,22 @@ export type JustReadFeedItemResolvers<ContextType = ResolverContext, ParentType
   author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   broadcastCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   comments?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
-  createdAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   dir?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   highlights?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
   languageCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   likedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  publishedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
   savedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
-  source?: Resolver<Maybe<ResolversTypes['JustReadFeedSource']>, ParentType, ContextType>;
+  siteName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  sourceIcon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  sourceName?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   topic?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  updatedAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  updatedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   wordCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -6105,7 +6108,7 @@ export type JustReadFeedSourceResolvers<ContextType = ResolverContext, ParentTyp
   name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   topics?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
-  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
@@ -6358,7 +6361,7 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
 };
 
 export type MySubscriptionRootTypeResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['MySubscriptionRootType'] = ResolversParentTypes['MySubscriptionRootType']> = {
-  justReadFeed?: SubscriptionResolver<ResolversTypes['JustReadFeedResult'], "justReadFeed", ParentType, ContextType>;
+  hello?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "hello", ParentType, ContextType>;
 };
 
 export type NewsletterEmailResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['NewsletterEmail'] = ResolversParentTypes['NewsletterEmail']> = {
@@ -6452,6 +6455,7 @@ export type QueryResolvers<ContextType = ResolverContext, ParentType extends Res
   hello?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   integration?: Resolver<ResolversTypes['IntegrationResult'], ParentType, ContextType, RequireFields<QueryIntegrationArgs, 'name'>>;
   integrations?: Resolver<ResolversTypes['IntegrationsResult'], ParentType, ContextType>;
+  justReadFeed?: Resolver<ResolversTypes['JustReadFeedResult'], ParentType, ContextType>;
   labels?: Resolver<ResolversTypes['LabelsResult'], ParentType, ContextType>;
   me?: Resolver<Maybe<ResolversTypes['User']>, ParentType, ContextType>;
   newsletterEmails?: Resolver<ResolversTypes['NewsletterEmailsResult'], ParentType, ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 0474deea6..c64e7c328 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1241,21 +1241,22 @@ type JustReadFeedItem {
   author: String
   broadcastCount: Int
   comments: [String!]
-  createdAt: Date
+  createdAt: Date!
   dir: String
   highlights: String
   id: ID!
   languageCode: String
   likedCount: Int
   previewContent: String
-  publishedAt: Date
   savedCount: Int
   seen_at: Date
-  source: JustReadFeedSource
+  siteName: String
+  sourceIcon: String
+  sourceName: String!
   thumbnail: String
   title: String!
   topic: String
-  updatedAt: Date
+  updatedAt: Date!
   url: String!
   wordCount: Int
 }
@@ -1268,7 +1269,7 @@ type JustReadFeedSource {
   name: String!
   thumbnail: String
   topics: [String!]
-  url: String
+  url: String!
 }
 
 type JustReadFeedSuccess {
@@ -1563,7 +1564,7 @@ type Mutation {
 }
 
 type MySubscriptionRootType {
-  justReadFeed: JustReadFeedResult!
+  hello: String
 }
 
 type NewsletterEmail {
@@ -1703,6 +1704,7 @@ type Query {
   hello: String
   integration(name: String!): IntegrationResult!
   integrations: IntegrationsResult!
+  justReadFeed: JustReadFeedResult!
   labels: LabelsResult!
   me: User
   newsletterEmails: NewsletterEmailsResult!
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index a72cf61de..54340d169 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -18,6 +18,7 @@ interface JustReadFeedItem {
   title: string
   url: string
   createdAt: Date
+  updatedAt: Date
   sourceName: string
 
   siteName?: string
@@ -59,6 +60,7 @@ const libraryItemToFeedItem = (
   sourceIcon: user.profile.pictureUrl || undefined,
   sourceName: user.name,
   siteName: item.siteName || undefined,
+  updatedAt: item.updatedAt,
 })
 
 const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
@@ -76,6 +78,7 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
   sourceIcon: item.sourceIcon,
   sourceName: item.sourceName,
   siteName: item.siteName,
+  updatedAt: item.updatedAt,
 })
 
 interface FeedItemScore {
@@ -97,20 +100,28 @@ const selectCandidates = async (
     userId
   )
 
+  logger.info(`Found ${libraryItems.length} library items`)
+
   // map library items to candidates
   const privateCandidates: Array<JustReadFeedItem> = libraryItems.map(
     (libraryItem) => libraryItemToFeedItem(user, libraryItem)
   )
 
+  logger.info(`Found ${privateCandidates.length} private candidates`)
+
   // get candidates from public inventory
   const publicItems = await findUnseenPublicItems(userId, {
     limit: 100,
   })
 
+  logger.info(`Found ${publicItems.length} public items`)
+
   // map public items to candidates
   const publicCandidates: Array<JustReadFeedItem> =
     publicItems.map(publicItemToFeedItem)
 
+  logger.info(`Found ${publicCandidates.length} public candidates`)
+
   // combine candidates
   return [...privateCandidates, ...publicCandidates]
 }
@@ -124,29 +135,37 @@ const rankFeedItems = async (
   }
 
   // TODO: get score of candidates
-  const API_URL = 'https://score.omnivore.app' // fake URL
+  // const API_URL = 'https://score.omnivore.app' // fake URL
 
-  const response = await fetch(API_URL, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify({ candidates: feedItems }),
-  })
+  // const response = await fetch(API_URL, {
+  //   method: 'POST',
+  //   headers: {
+  //     'Content-Type': 'application/json',
+  //   },
+  //   body: JSON.stringify({ candidates: feedItems }),
+  // })
 
-  if (!response.ok) {
-    throw new Error(`Failed to score candidates: ${response.statusText}`)
-  }
+  // if (!response.ok) {
+  //   throw new Error(`Failed to score candidates: ${response.statusText}`)
+  // }
 
-  const scores = (await response.json()) as Array<FeedItemScore>
+  // const scores = (await response.json()) as Array<FeedItemScore>
+
+  // fake scores
+  const scores: Array<FeedItemScore> = feedItems.map((item) => ({
+    id: item.id,
+    score: Math.random(),
+  }))
 
   // rank candidates by score in ascending order
-  return feedItems.sort((a, b) => {
+  feedItems.sort((a, b) => {
     const scoreA = scores.find((score) => score.id === a.id)?.score || 0
     const scoreB = scores.find((score) => score.id === b.id)?.score || 0
 
     return scoreA - scoreB
   })
+
+  return Promise.resolve(feedItems)
 }
 
 const redisKey = (userId: string) => `just-read-feed:${userId}`
@@ -209,6 +228,7 @@ const appendItemsToFeed = async (
   pipeline.zremrangebyrank(key, 0, -(MAX_FEED_ITEMS + 1))
   pipeline.zremrangebyscore(key, '-inf', Date.now())
 
+  logger.info('Adding feed items to redis')
   await pipeline.exec()
 }
 
@@ -227,11 +247,18 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
 
   // TODO: integrity check on candidates?
 
+  logger.info('Ranking feed items')
   const rankedFeedItems = await rankFeedItems(feedItems)
+  if (rankedFeedItems.length === 0) {
+    logger.info('No feed items to append')
+    return
+  }
 
+  logger.info('Filtering feed items')
   // TODO: filtering
   // get top 100 ranked feed items
   const filteredFeedItems = rankedFeedItems.slice(0, 100)
 
+  logger.info('Appending feed items to feed')
   await appendItemsToFeed(filteredFeedItems, userId)
 }
diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/just_read_feed/index.ts
index c907d1ddb..2343bd6b6 100644
--- a/packages/api/src/resolvers/just_read_feed/index.ts
+++ b/packages/api/src/resolvers/just_read_feed/index.ts
@@ -1,5 +1,6 @@
 import { JustReadFeedError, JustReadFeedSuccess } from '../../generated/graphql'
 import { getJustReadFeed } from '../../jobs/update_just_read_feed'
+import { enqueueUpdateJustReadFeed } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 
 export const justReadFeedResolver = authorized<
@@ -9,5 +10,11 @@ export const justReadFeedResolver = authorized<
   const feed = await getJustReadFeed(uid)
   log.info('Just read feed fetched')
 
+  if (feed.topics.length === 0) {
+    await enqueueUpdateJustReadFeed({
+      userId: uid,
+    })
+  }
+
   return feed
 })
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index ad6d04cbc..9220d141d 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3107,7 +3107,6 @@ const schema = gql`
     url: String!
     topics: [String!]
     thumbnail: String
-    url: String
     languageCodes: [String!]
   }
 
@@ -3117,21 +3116,22 @@ const schema = gql`
     topic: String
     url: String!
     thumbnail: String
-    publishedAt: Date
-    source: JustReadFeedSource
     previewContent: String
     highlights: String
     savedCount: Int
     likedCount: Int
     broadcastCount: Int
-    createdAt: Date
-    updatedAt: Date
+    createdAt: Date!
+    updatedAt: Date!
     comments: [String!]
     author: String
     languageCode: String
     dir: String
     seen_at: Date
     wordCount: Int
+    sourceName: String!
+    sourceIcon: String
+    siteName: String
   }
 
   type JustReadFeedTopic {
@@ -3156,7 +3156,7 @@ const schema = gql`
   union JustReadFeedResult = JustReadFeedSuccess | JustReadFeedError
 
   type MySubscriptionRootType {
-    justReadFeed: JustReadFeedResult!
+    hello: String # for testing only
   }
 
   # Mutations
@@ -3354,6 +3354,7 @@ const schema = gql`
     feeds(input: FeedsInput!): FeedsResult!
     discoverFeeds: DiscoverFeedResult!
     scanFeeds(input: ScanFeedsInput!): ScanFeedsResult!
+    justReadFeed: JustReadFeedResult!
   }
 
   schema {
diff --git a/packages/api/src/services/just_read_feed.ts b/packages/api/src/services/just_read_feed.ts
index c34fc7151..7d8a18103 100644
--- a/packages/api/src/services/just_read_feed.ts
+++ b/packages/api/src/services/just_read_feed.ts
@@ -1,5 +1,5 @@
 import { PublicItem } from '../entity/public_item'
-import { getRepository } from '../repository'
+import { authTrx } from '../repository'
 
 export const findUnseenPublicItems = async (
   userId: string,
@@ -7,17 +7,29 @@ export const findUnseenPublicItems = async (
     limit?: number
     offset?: number
   }
-): Promise<Array<PublicItem>> =>
-  getRepository(PublicItem)
-    .createQueryBuilder('public_item')
-    .leftJoin(
-      'omnivore.public_item_interactions',
-      'interaction',
-      'interaction.public_item_id = public_item.id'
-    )
-    .where('interaction.user_id = :userId', { userId })
-    .andWhere('interaction.seen_at IS NULL')
-    .orderBy('public_item.created_at', 'DESC')
-    .limit(options.limit)
-    .offset(options.offset)
-    .getMany()
+): Promise<Array<PublicItem>> => {
+  return authTrx(
+    async (tx) =>
+      tx
+        .getRepository(PublicItem)
+        .createQueryBuilder('public_item')
+        .leftJoin(
+          'public_item_interactions',
+          'interaction',
+          'interaction.public_item_id = public_item.id'
+        )
+        .innerJoin(
+          'public_item_stats',
+          'stats',
+          'stats.public_item_id = public_item.id'
+        )
+        .where('interaction.user_id = :userId', { userId })
+        .andWhere('interaction.seen_at IS NULL')
+        .orderBy('public_item.createdAt', 'DESC')
+        .take(options.limit)
+        .skip(options.offset)
+        .getMany(),
+    undefined,
+    userId
+  )
+}
diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index 59df6453a..32a618660 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -17,6 +17,7 @@ CREATE TABLE omnivore.public_item_source (
 );
 
 CREATE TRIGGER update_public_item_source_modtime BEFORE UPDATE ON omnivore.public_item_source FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
+GRANT SELECT ON omnivore.public_item_source TO omnivore_user;
 
 
 CREATE TABLE omnivore.public_item (
@@ -41,6 +42,7 @@ CREATE TABLE omnivore.public_item (
 );
 
 CREATE TRIGGER update_public_item_modtime BEFORE UPDATE ON omnivore.public_item FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
+GRANT SELECT ON omnivore.public_item TO omnivore_user;
 
 
 CREATE TABLE omnivore.public_item_stats (
@@ -55,6 +57,7 @@ CREATE TABLE omnivore.public_item_stats (
 
 CREATE INDEX public_item_stats_public_item_id_idx ON omnivore.public_item_stats(public_item_id);
 CREATE TRIGGER update_public_item_stats_modtime BEFORE UPDATE ON omnivore.public_item_stats FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
+GRANT SELECT ON omnivore.public_item_stats TO omnivore_user;
 
 
 CREATE TABLE omnivore.public_item_interactions (
@@ -66,13 +69,14 @@ CREATE TABLE omnivore.public_item_interactions (
     broadcasted_at TIMESTAMPTZ,
     seen_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
     digested_at TIMESTAMPTZ,
-    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
+    created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
 );
 
 CREATE INDEX public_item_interaction_user_id_idx ON omnivore.public_item_interactions(user_id);
 CREATE INDEX public_item_interaction_public_item_id_idx ON omnivore.public_item_interactions(public_item_id);
 CREATE TRIGGER update_public_item_interactions_modtime BEFORE UPDATE ON omnivore.public_item_interactions FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
+GRANT SELECT, INSERT, UPDATE ON omnivore.public_item_interactions TO omnivore_user;
 
 
 ALTER TABLE omnivore.library_item 

From 6c9f0c81f6cbe4622ff192430d74b761a948c998 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 24 May 2024 15:41:20 +0800
Subject: [PATCH 09/33] convert language to code

---
 packages/api/package.json                     |  1 +
 packages/api/src/entity/public_item_stats.ts  |  6 ++
 .../api/src/jobs/update_just_read_feed.ts     | 12 ++-
 yarn.lock                                     | 74 ++-----------------
 4 files changed, 21 insertions(+), 72 deletions(-)
 create mode 100644 packages/api/src/entity/public_item_stats.ts

diff --git a/packages/api/package.json b/packages/api/package.json
index c465b0fbe..d5fc669a8 100644
--- a/packages/api/package.json
+++ b/packages/api/package.json
@@ -15,6 +15,7 @@
   },
   "dependencies": {
     "@bmatei/apollo-prometheus-exporter": "^3.0.0",
+    "@cospired/i18n-iso-languages": "^4.2.0",
     "@google-cloud/logging-winston": "^6.0.0",
     "@google-cloud/monitoring": "^4.0.0",
     "@google-cloud/opentelemetry-cloud-trace-exporter": "^2.0.0",
diff --git a/packages/api/src/entity/public_item_stats.ts b/packages/api/src/entity/public_item_stats.ts
new file mode 100644
index 000000000..3e6a776cd
--- /dev/null
+++ b/packages/api/src/entity/public_item_stats.ts
@@ -0,0 +1,6 @@
+import { Entity } from 'typeorm'
+
+@Entity({ name: 'public_item_stats' })
+export class PublicItemStats {
+  
+}
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 54340d169..430a365ea 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -1,3 +1,4 @@
+import languages from '@cospired/i18n-iso-languages'
 import { LibraryItem } from '../entity/library_item'
 import { PublicItem } from '../entity/public_item'
 import { User } from '../entity/user'
@@ -42,6 +43,9 @@ interface JustReadFeed {
   topics: Array<JustReadFeedTopic>
 }
 
+const lanaugeToCode = (language: string): string =>
+  languages.getAlpha2Code(language, 'en') || 'en'
+
 const libraryItemToFeedItem = (
   user: User,
   item: LibraryItem
@@ -51,9 +55,9 @@ const libraryItemToFeedItem = (
   url: item.originalUrl,
   thumbnail: item.thumbnail || undefined,
   previewContent: item.description || undefined,
-  languageCode: item.itemLanguage || undefined, // TODO: map to language code
+  languageCode: lanaugeToCode(item.itemLanguage || 'English'),
   author: item.author || undefined,
-  dir: item.directionality || undefined,
+  dir: item.directionality || 'ltr',
   createdAt: item.createdAt,
   topic: item.topic,
   wordCount: item.wordCount || undefined,
@@ -69,9 +73,9 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
   url: item.url,
   thumbnail: item.thumbnail,
   previewContent: item.previewContent,
-  languageCode: item.languageCode,
+  languageCode: item.languageCode || 'en',
   author: item.author,
-  dir: item.dir,
+  dir: item.dir || 'ltr',
   createdAt: item.createdAt,
   topic: item.topic,
   wordCount: item.wordCount || undefined,
diff --git a/yarn.lock b/yarn.lock
index 688e36992..d1bde267b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2442,6 +2442,11 @@
   resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9"
   integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==
 
+"@cospired/i18n-iso-languages@^4.2.0":
+  version "4.2.0"
+  resolved "https://registry.yarnpkg.com/@cospired/i18n-iso-languages/-/i18n-iso-languages-4.2.0.tgz#094418a72f250fd612b3fc856b12f674a10864eb"
+  integrity sha512-vy8cq1176MTxVwB1X9niQjcIYOH29F8Huxtx8hLmT5Uz3l1ztGDGri8KN/4zE7LV2mCT7JrcAoNV/I9yb+lNUw==
+
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"
   resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1"
@@ -12870,11 +12875,6 @@ cookie-signature@1.0.6:
   resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
   integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw=
 
-cookie@0.4.0:
-  version "0.4.0"
-  resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba"
-  integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==
-
 cookie@0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.1.tgz#afd713fe26ebd21ba95ceb61f9a8116e50a537d1"
@@ -13175,15 +13175,6 @@ crypto@^1.0.1:
   resolved "https://registry.yarnpkg.com/crypto/-/crypto-1.0.1.tgz#2af1b7cad8175d24c8a1b0778255794a21803037"
   integrity sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig==
 
-csrf@3.1.0:
-  version "3.1.0"
-  resolved "https://registry.yarnpkg.com/csrf/-/csrf-3.1.0.tgz#ec75e9656d004d674b8ef5ba47b41fbfd6cb9c30"
-  integrity sha512-uTqEnCvWRk042asU6JtapDTcJeeailFy4ydOQS28bj1hcLnYRiqi8SsD2jS412AY1I/4qdOwWZun774iqywf9w==
-  dependencies:
-    rndm "1.2.0"
-    tsscmp "1.0.6"
-    uid-safe "2.1.5"
-
 css-loader@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/css-loader/-/css-loader-3.6.0.tgz#2e4b2c7e6e2d27f8c8f28f61bffcd2e6c91ef645"
@@ -13316,16 +13307,6 @@ csstype@^3.0.2:
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.8.tgz#d2266a792729fb227cd216fb572f43728e1ad340"
   integrity sha512-jXKhWqXPmlUeoQnF/EhTtTl4C9SnrxSH/jZUih3jmO6lBKr99rP3/+FmrMj4EFpOXzMtXHAZkd3x0E6h6Fgflw==
 
-csurf@^1.11.0:
-  version "1.11.0"
-  resolved "https://registry.yarnpkg.com/csurf/-/csurf-1.11.0.tgz#ab0c3c6634634192bd3d6f4b861be20800eeb61a"
-  integrity sha512-UCtehyEExKTxgiu8UHdGvHj4tnpE/Qctue03Giq5gPgMQ9cg/ciod5blZQ5a4uCEenNQjxyGuzygLdKUmee/bQ==
-  dependencies:
-    cookie "0.4.0"
-    cookie-signature "1.0.6"
-    csrf "3.1.0"
-    http-errors "~1.7.3"
-
 csv-file-validator@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/csv-file-validator/-/csv-file-validator-2.1.0.tgz#fc83e1e05835d7f03d03f8cce6235938e4cef32e"
@@ -17951,17 +17932,6 @@ http-errors@~1.6.2:
     setprototypeof "1.1.0"
     statuses ">= 1.4.0 < 2"
 
-http-errors@~1.7.3:
-  version "1.7.3"
-  resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06"
-  integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==
-  dependencies:
-    depd "~1.1.2"
-    inherits "2.0.4"
-    setprototypeof "1.1.1"
-    statuses ">= 1.5.0 < 2"
-    toidentifier "1.0.0"
-
 http-parser-js@>=0.5.1:
   version "0.5.5"
   resolved "https://registry.yarnpkg.com/http-parser-js/-/http-parser-js-0.5.5.tgz#d7c30d5d3c90d865b4a2e870181f9d6f22ac7ac5"
@@ -26218,11 +26188,6 @@ randexp@0.4.6:
     discontinuous-range "1.0.0"
     ret "~0.1.10"
 
-random-bytes@~1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/random-bytes/-/random-bytes-1.0.0.tgz#4f68a1dc0ae58bd3fb95848c30324db75d64360b"
-  integrity sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==
-
 randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5, randombytes@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a"
@@ -27837,11 +27802,6 @@ ripemd160@^2.0.0, ripemd160@^2.0.1:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
-rndm@1.2.0:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/rndm/-/rndm-1.2.0.tgz#f33fe9cfb52bbfd520aa18323bc65db110a1b76c"
-  integrity sha512-fJhQQI5tLrQvYIYFpOnFinzv9dwmR7hRnUz1XqP3OJ1jIweTNOd6aTO4jwQSgcBSFUB+/KHJxuGneime+FdzOw==
-
 rollup@2.78.0:
   version "2.78.0"
   resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.78.0.tgz#00995deae70c0f712ea79ad904d5f6b033209d9e"
@@ -28341,11 +28301,6 @@ setprototypeof@1.1.0:
   resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656"
   integrity sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==
 
-setprototypeof@1.1.1:
-  version "1.1.1"
-  resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683"
-  integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==
-
 setprototypeof@1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424"
@@ -28997,7 +28952,7 @@ statuses@2.0.1:
   resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63"
   integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==
 
-"statuses@>= 1.4.0 < 2", "statuses@>= 1.5.0 < 2":
+"statuses@>= 1.4.0 < 2":
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c"
   integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=
@@ -30075,11 +30030,6 @@ toggle-selection@^1.0.6:
   resolved "https://registry.yarnpkg.com/toggle-selection/-/toggle-selection-1.0.6.tgz#6e45b1263f2017fa0acc7d89d78b15b8bf77da32"
   integrity sha1-bkWxJj8gF/oKzH2J14sVuL932jI=
 
-toidentifier@1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553"
-  integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==
-
 toidentifier@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35"
@@ -30371,11 +30321,6 @@ tslib@~2.4.0:
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.1.tgz#0d0bfbaac2880b91e22df0768e55be9753a5b17e"
   integrity sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==
 
-tsscmp@1.0.6:
-  version "1.0.6"
-  resolved "https://registry.yarnpkg.com/tsscmp/-/tsscmp-1.0.6.tgz#85b99583ac3589ec4bfef825b5000aa911d605eb"
-  integrity sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==
-
 tsutils@^3.21.0:
   version "3.21.0"
   resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623"
@@ -30640,13 +30585,6 @@ uhyphen@^0.2.0:
   resolved "https://registry.yarnpkg.com/uhyphen/-/uhyphen-0.2.0.tgz#8fdf0623314486e020a3c00ee5cc7a12fe722b81"
   integrity sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==
 
-uid-safe@2.1.5:
-  version "2.1.5"
-  resolved "https://registry.yarnpkg.com/uid-safe/-/uid-safe-2.1.5.tgz#2b3d5c7240e8fc2e58f8aa269e5ee49c0857bd3a"
-  integrity sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==
-  dependencies:
-    random-bytes "~1.0.0"
-
 unbox-primitive@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471"

From 880a907679ed3664ca148eef53e50ef86026763b Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Fri, 24 May 2024 15:48:24 +0800
Subject: [PATCH 10/33] add public item stats

---
 packages/api/src/entity/public_item.ts        |  5 ++++
 packages/api/src/entity/public_item_stats.ts  | 29 +++++++++++++++++--
 .../api/src/jobs/update_just_read_feed.ts     |  7 +++++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts
index 0bf5c7862..c1f68cc34 100644
--- a/packages/api/src/entity/public_item.ts
+++ b/packages/api/src/entity/public_item.ts
@@ -2,15 +2,20 @@ import {
   Column,
   CreateDateColumn,
   Entity,
+  OneToOne,
   PrimaryGeneratedColumn,
   UpdateDateColumn,
 } from 'typeorm'
+import { PublicItemStats } from './public_item_stats'
 
 @Entity({ name: 'public_item' })
 export class PublicItem {
   @PrimaryGeneratedColumn('uuid')
   id!: string
 
+  @OneToOne(() => PublicItemStats)
+  stats!: PublicItemStats
+
   @Column('text')
   sourceName!: string
 
diff --git a/packages/api/src/entity/public_item_stats.ts b/packages/api/src/entity/public_item_stats.ts
index 3e6a776cd..ade4a5c25 100644
--- a/packages/api/src/entity/public_item_stats.ts
+++ b/packages/api/src/entity/public_item_stats.ts
@@ -1,6 +1,31 @@
-import { Entity } from 'typeorm'
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm'
 
 @Entity({ name: 'public_item_stats' })
 export class PublicItemStats {
-  
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @Column('uuid')
+  publicItemId!: string
+
+  @Column('integer')
+  saveCount!: number
+
+  @Column('integer')
+  likeCount!: number
+
+  @Column('integer')
+  broadcastCount!: number
+
+  @CreateDateColumn()
+  createdAt!: Date
+
+  @UpdateDateColumn()
+  updatedAt!: Date
 }
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 430a365ea..2bbdec0a1 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -31,6 +31,10 @@ interface JustReadFeedItem {
   dir?: string
   wordCount?: number
   sourceIcon?: string
+
+  saveCount?: number
+  likeCount?: number
+  broadcastCount?: number
 }
 
 interface JustReadFeedTopic {
@@ -83,6 +87,9 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
   sourceName: item.sourceName,
   siteName: item.siteName,
   updatedAt: item.updatedAt,
+  saveCount: item.stats.saveCount,
+  likeCount: item.stats.likeCount,
+  broadcastCount: item.stats.broadcastCount,
 })
 
 interface FeedItemScore {

From 1ba6221bc4559c990b30b129c5840e06b471a5e2 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sat, 25 May 2024 11:47:45 +0800
Subject: [PATCH 11/33] add pagination

---
 packages/api/src/generated/graphql.ts         | 131 ++++++-------
 packages/api/src/generated/schema.graphql     |  53 +++---
 .../api/src/jobs/update_just_read_feed.ts     | 176 ++++++++++--------
 .../api/src/resolvers/just_read_feed/index.ts |  18 +-
 packages/api/src/schema.ts                    |  37 ++--
 5 files changed, 219 insertions(+), 196 deletions(-)

diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 514a7d2ba..ed120561a 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1370,6 +1370,7 @@ export type JustReadFeedError = {
 
 export enum JustReadFeedErrorCode {
   BadRequest = 'BAD_REQUEST',
+  Pending = 'PENDING',
   Unauthorized = 'UNAUTHORIZED'
 }
 
@@ -1377,49 +1378,47 @@ export type JustReadFeedItem = {
   __typename?: 'JustReadFeedItem';
   author?: Maybe<Scalars['String']>;
   broadcastCount?: Maybe<Scalars['Int']>;
-  comments?: Maybe<Array<Scalars['String']>>;
-  createdAt: Scalars['Date'];
+  canArchive?: Maybe<Scalars['Boolean']>;
+  canComment?: Maybe<Scalars['Boolean']>;
+  canDelete?: Maybe<Scalars['Boolean']>;
+  canSave?: Maybe<Scalars['Boolean']>;
+  canShare?: Maybe<Scalars['Boolean']>;
+  date: Scalars['Date'];
   dir?: Maybe<Scalars['String']>;
   highlights?: Maybe<Scalars['String']>;
   id: Scalars['ID'];
-  languageCode?: Maybe<Scalars['String']>;
   likedCount?: Maybe<Scalars['Int']>;
   previewContent?: Maybe<Scalars['String']>;
   savedCount?: Maybe<Scalars['Int']>;
   seen_at?: Maybe<Scalars['Date']>;
-  siteName?: Maybe<Scalars['String']>;
-  sourceIcon?: Maybe<Scalars['String']>;
-  sourceName: Scalars['String'];
+  subscription: JustReadFeedSubscription;
   thumbnail?: Maybe<Scalars['String']>;
   title: Scalars['String'];
-  topic?: Maybe<Scalars['String']>;
-  updatedAt: Scalars['Date'];
   url: Scalars['String'];
   wordCount?: Maybe<Scalars['Int']>;
 };
 
 export type JustReadFeedResult = JustReadFeedError | JustReadFeedSuccess;
 
-export type JustReadFeedSource = {
-  __typename?: 'JustReadFeedSource';
-  id: Scalars['ID'];
-  languageCodes?: Maybe<Array<Scalars['String']>>;
-  name: Scalars['String'];
+export type JustReadFeedSection = {
+  __typename?: 'JustReadFeedSection';
+  items: Array<JustReadFeedItem>;
+  layout?: Maybe<Scalars['String']>;
   thumbnail?: Maybe<Scalars['String']>;
-  topics?: Maybe<Array<Scalars['String']>>;
-  url: Scalars['String'];
+  title?: Maybe<Scalars['String']>;
+};
+
+export type JustReadFeedSubscription = {
+  __typename?: 'JustReadFeedSubscription';
+  icon?: Maybe<Scalars['String']>;
+  id: Scalars['ID'];
+  name: Scalars['String'];
+  url?: Maybe<Scalars['String']>;
 };
 
 export type JustReadFeedSuccess = {
   __typename?: 'JustReadFeedSuccess';
-  topics: Array<JustReadFeedTopic>;
-};
-
-export type JustReadFeedTopic = {
-  __typename?: 'JustReadFeedTopic';
-  items: Array<JustReadFeedItem>;
-  name?: Maybe<Scalars['String']>;
-  thumbnail?: Maybe<Scalars['String']>;
+  sections: Array<JustReadFeedSection>;
 };
 
 export type Label = {
@@ -2074,11 +2073,6 @@ export type MutationUploadImportFileArgs = {
   type: UploadImportFileType;
 };
 
-export type MySubscriptionRootType = {
-  __typename?: 'MySubscriptionRootType';
-  hello?: Maybe<Scalars['String']>;
-};
-
 export type NewsletterEmail = {
   __typename?: 'NewsletterEmail';
   address: Scalars['String'];
@@ -2277,6 +2271,12 @@ export type QueryIntegrationArgs = {
 };
 
 
+export type QueryJustReadFeedArgs = {
+  after?: InputMaybe<Scalars['String']>;
+  first?: InputMaybe<Scalars['Int']>;
+};
+
+
 export type QueryRulesArgs = {
   enabled?: InputMaybe<Scalars['Boolean']>;
 };
@@ -3285,6 +3285,11 @@ export type Subscription = {
   url?: Maybe<Scalars['String']>;
 };
 
+export type SubscriptionRootType = {
+  __typename?: 'SubscriptionRootType';
+  hello?: Maybe<Scalars['String']>;
+};
+
 export enum SubscriptionStatus {
   Active = 'ACTIVE',
   Deleted = 'DELETED',
@@ -4270,9 +4275,9 @@ export type ResolversTypes = {
   JustReadFeedErrorCode: JustReadFeedErrorCode;
   JustReadFeedItem: ResolverTypeWrapper<JustReadFeedItem>;
   JustReadFeedResult: ResolversTypes['JustReadFeedError'] | ResolversTypes['JustReadFeedSuccess'];
-  JustReadFeedSource: ResolverTypeWrapper<JustReadFeedSource>;
+  JustReadFeedSection: ResolverTypeWrapper<JustReadFeedSection>;
+  JustReadFeedSubscription: ResolverTypeWrapper<JustReadFeedSubscription>;
   JustReadFeedSuccess: ResolverTypeWrapper<JustReadFeedSuccess>;
-  JustReadFeedTopic: ResolverTypeWrapper<JustReadFeedTopic>;
   Label: ResolverTypeWrapper<Label>;
   LabelsError: ResolverTypeWrapper<LabelsError>;
   LabelsErrorCode: LabelsErrorCode;
@@ -4316,7 +4321,6 @@ export type ResolversTypes = {
   MoveToFolderResult: ResolversTypes['MoveToFolderError'] | ResolversTypes['MoveToFolderSuccess'];
   MoveToFolderSuccess: ResolverTypeWrapper<MoveToFolderSuccess>;
   Mutation: ResolverTypeWrapper<{}>;
-  MySubscriptionRootType: ResolverTypeWrapper<{}>;
   NewsletterEmail: ResolverTypeWrapper<NewsletterEmail>;
   NewsletterEmailsError: ResolverTypeWrapper<NewsletterEmailsError>;
   NewsletterEmailsErrorCode: NewsletterEmailsErrorCode;
@@ -4495,6 +4499,7 @@ export type ResolversTypes = {
   SubscribeResult: ResolversTypes['SubscribeError'] | ResolversTypes['SubscribeSuccess'];
   SubscribeSuccess: ResolverTypeWrapper<SubscribeSuccess>;
   Subscription: ResolverTypeWrapper<Subscription>;
+  SubscriptionRootType: ResolverTypeWrapper<{}>;
   SubscriptionStatus: SubscriptionStatus;
   SubscriptionType: SubscriptionType;
   SubscriptionsError: ResolverTypeWrapper<SubscriptionsError>;
@@ -4819,9 +4824,9 @@ export type ResolversParentTypes = {
   JustReadFeedError: JustReadFeedError;
   JustReadFeedItem: JustReadFeedItem;
   JustReadFeedResult: ResolversParentTypes['JustReadFeedError'] | ResolversParentTypes['JustReadFeedSuccess'];
-  JustReadFeedSource: JustReadFeedSource;
+  JustReadFeedSection: JustReadFeedSection;
+  JustReadFeedSubscription: JustReadFeedSubscription;
   JustReadFeedSuccess: JustReadFeedSuccess;
-  JustReadFeedTopic: JustReadFeedTopic;
   Label: Label;
   LabelsError: LabelsError;
   LabelsResult: ResolversParentTypes['LabelsError'] | ResolversParentTypes['LabelsSuccess'];
@@ -4856,7 +4861,6 @@ export type ResolversParentTypes = {
   MoveToFolderResult: ResolversParentTypes['MoveToFolderError'] | ResolversParentTypes['MoveToFolderSuccess'];
   MoveToFolderSuccess: MoveToFolderSuccess;
   Mutation: {};
-  MySubscriptionRootType: {};
   NewsletterEmail: NewsletterEmail;
   NewsletterEmailsError: NewsletterEmailsError;
   NewsletterEmailsResult: ResolversParentTypes['NewsletterEmailsError'] | ResolversParentTypes['NewsletterEmailsSuccess'];
@@ -4996,6 +5000,7 @@ export type ResolversParentTypes = {
   SubscribeResult: ResolversParentTypes['SubscribeError'] | ResolversParentTypes['SubscribeSuccess'];
   SubscribeSuccess: SubscribeSuccess;
   Subscription: Subscription;
+  SubscriptionRootType: {};
   SubscriptionsError: SubscriptionsError;
   SubscriptionsResult: ResolversParentTypes['SubscriptionsError'] | ResolversParentTypes['SubscriptionsSuccess'];
   SubscriptionsSuccess: SubscriptionsSuccess;
@@ -6076,23 +6081,22 @@ export type JustReadFeedErrorResolvers<ContextType = ResolverContext, ParentType
 export type JustReadFeedItemResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedItem'] = ResolversParentTypes['JustReadFeedItem']> = {
   author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   broadcastCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
-  comments?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
-  createdAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
+  canArchive?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  canComment?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  canDelete?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  canSave?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  canShare?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  date?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   dir?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   highlights?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
-  languageCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   likedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   savedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
-  siteName?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  sourceIcon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  sourceName?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  subscription?: Resolver<ResolversTypes['JustReadFeedSubscription'], ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  topic?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  updatedAt?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   wordCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -6102,25 +6106,24 @@ export type JustReadFeedResultResolvers<ContextType = ResolverContext, ParentTyp
   __resolveType: TypeResolveFn<'JustReadFeedError' | 'JustReadFeedSuccess', ParentType, ContextType>;
 };
 
-export type JustReadFeedSourceResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSource'] = ResolversParentTypes['JustReadFeedSource']> = {
-  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
-  languageCodes?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
-  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+export type JustReadFeedSectionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSection'] = ResolversParentTypes['JustReadFeedSection']> = {
+  items?: Resolver<Array<ResolversTypes['JustReadFeedItem']>, ParentType, ContextType>;
+  layout?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  topics?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType>;
-  url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  title?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type JustReadFeedSubscriptionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSubscription'] = ResolversParentTypes['JustReadFeedSubscription']> = {
+  icon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
 export type JustReadFeedSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSuccess'] = ResolversParentTypes['JustReadFeedSuccess']> = {
-  topics?: Resolver<Array<ResolversTypes['JustReadFeedTopic']>, ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
-export type JustReadFeedTopicResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedTopic'] = ResolversParentTypes['JustReadFeedTopic']> = {
-  items?: Resolver<Array<ResolversTypes['JustReadFeedItem']>, ParentType, ContextType>;
-  name?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  sections?: Resolver<Array<ResolversTypes['JustReadFeedSection']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
@@ -6360,10 +6363,6 @@ export type MutationResolvers<ContextType = ResolverContext, ParentType extends
   uploadImportFile?: Resolver<ResolversTypes['UploadImportFileResult'], ParentType, ContextType, RequireFields<MutationUploadImportFileArgs, 'contentType' | 'type'>>;
 };
 
-export type MySubscriptionRootTypeResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['MySubscriptionRootType'] = ResolversParentTypes['MySubscriptionRootType']> = {
-  hello?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "hello", ParentType, ContextType>;
-};
-
 export type NewsletterEmailResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['NewsletterEmail'] = ResolversParentTypes['NewsletterEmail']> = {
   address?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   confirmationCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
@@ -6455,7 +6454,7 @@ export type QueryResolvers<ContextType = ResolverContext, ParentType extends Res
   hello?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   integration?: Resolver<ResolversTypes['IntegrationResult'], ParentType, ContextType, RequireFields<QueryIntegrationArgs, 'name'>>;
   integrations?: Resolver<ResolversTypes['IntegrationsResult'], ParentType, ContextType>;
-  justReadFeed?: Resolver<ResolversTypes['JustReadFeedResult'], ParentType, ContextType>;
+  justReadFeed?: Resolver<ResolversTypes['JustReadFeedResult'], ParentType, ContextType, Partial<QueryJustReadFeedArgs>>;
   labels?: Resolver<ResolversTypes['LabelsResult'], ParentType, ContextType>;
   me?: Resolver<Maybe<ResolversTypes['User']>, ParentType, ContextType>;
   newsletterEmails?: Resolver<ResolversTypes['NewsletterEmailsResult'], ParentType, ContextType>;
@@ -7064,6 +7063,10 @@ export type SubscriptionResolvers<ContextType = ResolverContext, ParentType exte
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type SubscriptionRootTypeResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['SubscriptionRootType'] = ResolversParentTypes['SubscriptionRootType']> = {
+  hello?: SubscriptionResolver<Maybe<ResolversTypes['String']>, "hello", ParentType, ContextType>;
+};
+
 export type SubscriptionsErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['SubscriptionsError'] = ResolversParentTypes['SubscriptionsError']> = {
   errorCodes?: Resolver<Array<ResolversTypes['SubscriptionsErrorCode']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -7650,9 +7653,9 @@ export type Resolvers<ContextType = ResolverContext> = {
   JustReadFeedError?: JustReadFeedErrorResolvers<ContextType>;
   JustReadFeedItem?: JustReadFeedItemResolvers<ContextType>;
   JustReadFeedResult?: JustReadFeedResultResolvers<ContextType>;
-  JustReadFeedSource?: JustReadFeedSourceResolvers<ContextType>;
+  JustReadFeedSection?: JustReadFeedSectionResolvers<ContextType>;
+  JustReadFeedSubscription?: JustReadFeedSubscriptionResolvers<ContextType>;
   JustReadFeedSuccess?: JustReadFeedSuccessResolvers<ContextType>;
-  JustReadFeedTopic?: JustReadFeedTopicResolvers<ContextType>;
   Label?: LabelResolvers<ContextType>;
   LabelsError?: LabelsErrorResolvers<ContextType>;
   LabelsResult?: LabelsResultResolvers<ContextType>;
@@ -7684,7 +7687,6 @@ export type Resolvers<ContextType = ResolverContext> = {
   MoveToFolderResult?: MoveToFolderResultResolvers<ContextType>;
   MoveToFolderSuccess?: MoveToFolderSuccessResolvers<ContextType>;
   Mutation?: MutationResolvers<ContextType>;
-  MySubscriptionRootType?: MySubscriptionRootTypeResolvers<ContextType>;
   NewsletterEmail?: NewsletterEmailResolvers<ContextType>;
   NewsletterEmailsError?: NewsletterEmailsErrorResolvers<ContextType>;
   NewsletterEmailsResult?: NewsletterEmailsResultResolvers<ContextType>;
@@ -7795,6 +7797,7 @@ export type Resolvers<ContextType = ResolverContext> = {
   SubscribeResult?: SubscribeResultResolvers<ContextType>;
   SubscribeSuccess?: SubscribeSuccessResolvers<ContextType>;
   Subscription?: SubscriptionResolvers<ContextType>;
+  SubscriptionRootType?: SubscriptionRootTypeResolvers<ContextType>;
   SubscriptionsError?: SubscriptionsErrorResolvers<ContextType>;
   SubscriptionsResult?: SubscriptionsResultResolvers<ContextType>;
   SubscriptionsSuccess?: SubscriptionsSuccessResolvers<ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index c64e7c328..8899e679a 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1,7 +1,7 @@
 schema {
   query: Query
   mutation: Mutation
-  subscription: MySubscriptionRootType
+  subscription: SubscriptionRootType
 }
 
 directive @sanitize(allowedTags: [String], maxLength: Int, minLength: Int, pattern: String) on INPUT_FIELD_DEFINITION
@@ -1234,52 +1234,51 @@ type JustReadFeedError {
 
 enum JustReadFeedErrorCode {
   BAD_REQUEST
+  PENDING
   UNAUTHORIZED
 }
 
 type JustReadFeedItem {
   author: String
   broadcastCount: Int
-  comments: [String!]
-  createdAt: Date!
+  canArchive: Boolean
+  canComment: Boolean
+  canDelete: Boolean
+  canSave: Boolean
+  canShare: Boolean
+  date: Date!
   dir: String
   highlights: String
   id: ID!
-  languageCode: String
   likedCount: Int
   previewContent: String
   savedCount: Int
   seen_at: Date
-  siteName: String
-  sourceIcon: String
-  sourceName: String!
+  subscription: JustReadFeedSubscription!
   thumbnail: String
   title: String!
-  topic: String
-  updatedAt: Date!
   url: String!
   wordCount: Int
 }
 
 union JustReadFeedResult = JustReadFeedError | JustReadFeedSuccess
 
-type JustReadFeedSource {
-  id: ID!
-  languageCodes: [String!]
-  name: String!
+type JustReadFeedSection {
+  items: [JustReadFeedItem!]!
+  layout: String
   thumbnail: String
-  topics: [String!]
-  url: String!
+  title: String
+}
+
+type JustReadFeedSubscription {
+  icon: String
+  id: ID!
+  name: String!
+  url: String
 }
 
 type JustReadFeedSuccess {
-  topics: [JustReadFeedTopic!]!
-}
-
-type JustReadFeedTopic {
-  items: [JustReadFeedItem!]!
-  name: String
-  thumbnail: String
+  sections: [JustReadFeedSection!]!
 }
 
 type Label {
@@ -1563,10 +1562,6 @@ type Mutation {
   uploadImportFile(contentType: String!, type: UploadImportFileType!): UploadImportFileResult!
 }
 
-type MySubscriptionRootType {
-  hello: String
-}
-
 type NewsletterEmail {
   address: String!
   confirmationCode: String
@@ -1704,7 +1699,7 @@ type Query {
   hello: String
   integration(name: String!): IntegrationResult!
   integrations: IntegrationsResult!
-  justReadFeed: JustReadFeedResult!
+  justReadFeed(after: String, first: Int): JustReadFeedResult!
   labels: LabelsResult!
   me: User
   newsletterEmails: NewsletterEmailsResult!
@@ -2606,6 +2601,10 @@ type Subscription {
   url: String
 }
 
+type SubscriptionRootType {
+  hello: String
+}
+
 enum SubscriptionStatus {
   ACTIVE
   DELETED
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 2bbdec0a1..30d5bf6ff 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -2,6 +2,11 @@ import languages from '@cospired/i18n-iso-languages'
 import { LibraryItem } from '../entity/library_item'
 import { PublicItem } from '../entity/public_item'
 import { User } from '../entity/user'
+import {
+  JustReadFeedItem,
+  JustReadFeedSection,
+  JustReadFeedSuccess,
+} from '../generated/graphql'
 import { redisDataSource } from '../redis_data_source'
 import { findUnseenPublicItems } from '../services/just_read_feed'
 import { searchLibraryItems } from '../services/library_item'
@@ -14,37 +19,44 @@ export interface UpdateJustReadFeedJobData {
   userId: string
 }
 
-interface JustReadFeedItem {
+interface Feature {
+  title: string
+  has_thumbnail: boolean
+  has_site_icon: boolean
+  saved_at: Date
+  site?: string
+  language?: string
+  author?: string
+  directionality: string
+  word_count?: number
+  subscription_type: string
+  folder: string
+}
+
+interface ScoreApiRequestBody {
+  user_id: string
+  item_features: Record<string, Feature> // item_id -> feature
+}
+
+interface Candidate {
   id: string
   title: string
   url: string
-  createdAt: Date
-  updatedAt: Date
-  sourceName: string
-
-  siteName?: string
-  topic?: string
   thumbnail?: string
   previewContent?: string
-  languageCode?: string
+  languageCode: string
   author?: string
-  dir?: string
+  dir: string
+  date: Date
+  topic?: string
   wordCount?: number
   sourceIcon?: string
-
-  saveCount?: number
-  likeCount?: number
-  broadcastCount?: number
-}
-
-interface JustReadFeedTopic {
-  name?: string
-  items: Array<JustReadFeedItem>
-  thumbnail: string
-}
-
-interface JustReadFeed {
-  topics: Array<JustReadFeedTopic>
+  siteName?: string
+  subscription: {
+    id: string
+    name: string
+    icon?: string
+  }
 }
 
 const lanaugeToCode = (language: string): string =>
@@ -62,13 +74,16 @@ const libraryItemToFeedItem = (
   languageCode: lanaugeToCode(item.itemLanguage || 'English'),
   author: item.author || undefined,
   dir: item.directionality || 'ltr',
-  createdAt: item.createdAt,
+  date: item.createdAt,
   topic: item.topic,
   wordCount: item.wordCount || undefined,
   sourceIcon: user.profile.pictureUrl || undefined,
-  sourceName: user.name,
   siteName: item.siteName || undefined,
-  updatedAt: item.updatedAt,
+  subscription: {
+    id: user.id,
+    name: user.name,
+    icon: user.profile.pictureUrl || undefined,
+  },
 })
 
 const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
@@ -80,11 +95,9 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
   languageCode: item.languageCode || 'en',
   author: item.author,
   dir: item.dir || 'ltr',
-  createdAt: item.createdAt,
+  date: item.createdAt,
   topic: item.topic,
   wordCount: item.wordCount || undefined,
-  sourceIcon: item.sourceIcon,
-  sourceName: item.sourceName,
   siteName: item.siteName,
   updatedAt: item.updatedAt,
   saveCount: item.stats.saveCount,
@@ -92,10 +105,7 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
   broadcastCount: item.stats.broadcastCount,
 })
 
-interface FeedItemScore {
-  id: string
-  score: number
-}
+type ScoreApiResponse = Record<string, number> // item_id -> score
 
 const selectCandidates = async (
   user: User
@@ -138,6 +148,7 @@ const selectCandidates = async (
 }
 
 const rankFeedItems = async (
+  userId: string,
   feedItems: Array<JustReadFeedItem>
 ): Promise<Array<JustReadFeedItem>> => {
   if (feedItems.length <= 10) {
@@ -146,32 +157,46 @@ const rankFeedItems = async (
   }
 
   // TODO: get score of candidates
-  // const API_URL = 'https://score.omnivore.app' // fake URL
+  const API_URL = 'http://127.0.0.1:5000/predictions'
+  const requestBody: ScoreApiRequestBody = {
+    user_id: userId,
+    item_features: feedItems.reduce((acc, item) => {
+      acc[item.id] = {
+        title: item.title,
+        has_thumbnail: !!item.thumbnail,
+        has_site_icon: !!item.sourceIcon,
+        saved_at: item.date,
+        site: item.siteName,
+        language: item.languageCode,
+      } as Feature
+      return acc
+    }, {} as Record<string, Feature>),
+  }
 
-  // const response = await fetch(API_URL, {
-  //   method: 'POST',
-  //   headers: {
-  //     'Content-Type': 'application/json',
-  //   },
-  //   body: JSON.stringify({ candidates: feedItems }),
-  // })
+  const response = await fetch(API_URL, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(requestBody),
+  })
 
-  // if (!response.ok) {
-  //   throw new Error(`Failed to score candidates: ${response.statusText}`)
-  // }
+  if (!response.ok) {
+    throw new Error(`Failed to score candidates: ${response.statusText}`)
+  }
 
-  // const scores = (await response.json()) as Array<FeedItemScore>
+  const scores = (await response.json()) as ScoreApiResponse
 
-  // fake scores
-  const scores: Array<FeedItemScore> = feedItems.map((item) => ({
-    id: item.id,
-    score: Math.random(),
-  }))
+  // // fake scores
+  // const scores: Array<FeedItemScore> = feedItems.map((item) => ({
+  //   id: item.id,
+  //   score: Math.random(),
+  // }))
 
   // rank candidates by score in ascending order
   feedItems.sort((a, b) => {
-    const scoreA = scores.find((score) => score.id === a.id)?.score || 0
-    const scoreB = scores.find((score) => score.id === b.id)?.score || 0
+    const scoreA = scores[a.id] || 0
+    const scoreB = scores[b.id] || 0
 
     return scoreA - scoreB
   })
@@ -183,8 +208,10 @@ const redisKey = (userId: string) => `just-read-feed:${userId}`
 const MAX_FEED_ITEMS = 500
 
 export const getJustReadFeed = async (
-  userId: string
-): Promise<JustReadFeed> => {
+  userId: string,
+  first = 10,
+  after?: string
+): Promise<JustReadFeedSuccess> => {
   const redisClient = redisDataSource.redisClient
   if (!redisClient) {
     throw new Error('Redis client not available')
@@ -192,31 +219,16 @@ export const getJustReadFeed = async (
 
   const key = redisKey(userId)
 
-  const results = await redisClient.zrevrange(key, 0, MAX_FEED_ITEMS)
+  const results = await redisClient.zrevrange(key, 0, first - 1, 'WITHSCORES')
 
-  const feedItems = results.map((item) => JSON.parse(item) as JustReadFeedItem)
+  const sections = JSON.parse(results.join(',')) as JustReadFeedSection[]
 
-  const topics: Array<JustReadFeedTopic> = []
-
-  feedItems.forEach((item) => {
-    const topic = topics.find((topic) => topic.name === item.topic)
-    if (topic) {
-      topic.items.push(item)
-    } else {
-      topics.push({
-        name: item.topic,
-        thumbnail: item.thumbnail || '',
-        items: [item],
-      })
-    }
-  })
-
-  return { topics }
+  return { sections }
 }
 
-const appendItemsToFeed = async (
-  feedItems: Array<JustReadFeedItem>,
-  userId: string
+const appendSectionsToFeed = async (
+  userId: string,
+  sections: Array<JustReadFeedSection>
 ) => {
   const redisClient = redisDataSource.redisClient
   if (!redisClient) {
@@ -228,18 +240,18 @@ const appendItemsToFeed = async (
   // store candidates in redis sorted set
   const pipeline = redisClient.pipeline()
 
-  const scoreMembers = feedItems.flatMap((item) => [
-    Date.now() + 86_400_000, // items expire in 24 hours
+  const scoreMembers = sections.flatMap((item) => [
+    Date.now() + 86_400_000, // sections expire in 24 hours
     JSON.stringify(item),
   ])
-  // add candidates to the sorted set
+  // add section to the sorted set
   pipeline.zadd(key, ...scoreMembers)
 
-  // remove expired items and keep only the top 500
+  // remove expired sections and keep only the top 500
   pipeline.zremrangebyrank(key, 0, -(MAX_FEED_ITEMS + 1))
   pipeline.zremrangebyscore(key, '-inf', Date.now())
 
-  logger.info('Adding feed items to redis')
+  logger.info('Adding feed sections to redis')
   await pipeline.exec()
 }
 
@@ -259,7 +271,7 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
   // TODO: integrity check on candidates?
 
   logger.info('Ranking feed items')
-  const rankedFeedItems = await rankFeedItems(feedItems)
+  const rankedFeedItems = await rankFeedItems(userId, feedItems)
   if (rankedFeedItems.length === 0) {
     logger.info('No feed items to append')
     return
@@ -271,5 +283,5 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
   const filteredFeedItems = rankedFeedItems.slice(0, 100)
 
   logger.info('Appending feed items to feed')
-  await appendItemsToFeed(filteredFeedItems, userId)
+  await appendSectionsToFeed(userId, filteredFeedItems)
 }
diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/just_read_feed/index.ts
index 2343bd6b6..9b03219ad 100644
--- a/packages/api/src/resolvers/just_read_feed/index.ts
+++ b/packages/api/src/resolvers/just_read_feed/index.ts
@@ -1,4 +1,8 @@
-import { JustReadFeedError, JustReadFeedSuccess } from '../../generated/graphql'
+import {
+  JustReadFeedError,
+  JustReadFeedErrorCode,
+  JustReadFeedSuccess
+} from '../../generated/graphql'
 import { getJustReadFeed } from '../../jobs/update_just_read_feed'
 import { enqueueUpdateJustReadFeed } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
@@ -6,14 +10,20 @@ import { authorized } from '../../utils/gql-utils'
 export const justReadFeedResolver = authorized<
   JustReadFeedSuccess,
   JustReadFeedError
->(async (_, __, { uid, log }) => {
-  const feed = await getJustReadFeed(uid)
+>(async (_, { first, after }, { uid, log }) => {
+  const feed = await getJustReadFeed(uid, first, after)
   log.info('Just read feed fetched')
 
-  if (feed.topics.length === 0) {
+  if (feed.sections.length === 0) {
     await enqueueUpdateJustReadFeed({
       userId: uid,
     })
+
+    log.info('Just read feed update enqueued')
+
+    return {
+      errorCodes: [JustReadFeedErrorCode.Pending],
+    }
   }
 
   return feed
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 9220d141d..2c8d7e3aa 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3101,19 +3101,16 @@ const schema = gql`
     SUBSCRIBE
   }
 
-  type JustReadFeedSource {
+  type JustReadFeedSubscription {
     id: ID!
     name: String!
-    url: String!
-    topics: [String!]
-    thumbnail: String
-    languageCodes: [String!]
+    url: String
+    icon: String
   }
 
   type JustReadFeedItem {
     id: ID!
     title: String!
-    topic: String
     url: String!
     thumbnail: String
     previewContent: String
@@ -3121,32 +3118,34 @@ const schema = gql`
     savedCount: Int
     likedCount: Int
     broadcastCount: Int
-    createdAt: Date!
-    updatedAt: Date!
-    comments: [String!]
+    date: Date!
     author: String
-    languageCode: String
     dir: String
     seen_at: Date
     wordCount: Int
-    sourceName: String!
-    sourceIcon: String
-    siteName: String
+    subscription: JustReadFeedSubscription!
+    canSave: Boolean
+    canComment: Boolean
+    canShare: Boolean
+    canArchive: Boolean
+    canDelete: Boolean
   }
 
-  type JustReadFeedTopic {
-    name: String
+  type JustReadFeedSection {
+    title: String
+    layout: String
     items: [JustReadFeedItem!]!
     thumbnail: String
   }
 
   type JustReadFeedSuccess {
-    topics: [JustReadFeedTopic!]!
+    sections: [JustReadFeedSection!]!
   }
 
   enum JustReadFeedErrorCode {
     UNAUTHORIZED
     BAD_REQUEST
+    PENDING
   }
 
   type JustReadFeedError {
@@ -3155,7 +3154,7 @@ const schema = gql`
 
   union JustReadFeedResult = JustReadFeedSuccess | JustReadFeedError
 
-  type MySubscriptionRootType {
+  type SubscriptionRootType {
     hello: String # for testing only
   }
 
@@ -3354,13 +3353,13 @@ const schema = gql`
     feeds(input: FeedsInput!): FeedsResult!
     discoverFeeds: DiscoverFeedResult!
     scanFeeds(input: ScanFeedsInput!): ScanFeedsResult!
-    justReadFeed: JustReadFeedResult!
+    justReadFeed(first: Int, after: String): JustReadFeedResult!
   }
 
   schema {
     query: Query
     mutation: Mutation
-    subscription: MySubscriptionRootType
+    subscription: SubscriptionRootType
   }
 `
 

From 0a8cc13ac5637ba90e8127b463682bfc6d94801f Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sun, 26 May 2024 14:31:22 +0800
Subject: [PATCH 12/33] add mixing algorithm

---
 packages/api/src/entity/public_item.ts        |   5 +-
 packages/api/src/generated/graphql.ts         |  21 +-
 packages/api/src/generated/schema.graphql     |   8 +-
 .../api/src/jobs/update_just_read_feed.ts     | 242 +++++++++++++-----
 .../api/src/resolvers/just_read_feed/index.ts |  33 ++-
 packages/api/src/schema.ts                    |   8 +-
 6 files changed, 237 insertions(+), 80 deletions(-)

diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts
index c1f68cc34..e2843a399 100644
--- a/packages/api/src/entity/public_item.ts
+++ b/packages/api/src/entity/public_item.ts
@@ -17,10 +17,7 @@ export class PublicItem {
   stats!: PublicItemStats
 
   @Column('text')
-  sourceName!: string
-
-  @Column('text')
-  sourceIcon?: string
+  siteIcon?: string
 
   @Column('text')
   type!: string
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index ed120561a..b3e7b91c3 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1363,6 +1363,12 @@ export type JoinGroupSuccess = {
   group: RecommendationGroup;
 };
 
+export type JustReadFeedEdge = {
+  __typename?: 'JustReadFeedEdge';
+  cursor: Scalars['String'];
+  node: JustReadFeedSection;
+};
+
 export type JustReadFeedError = {
   __typename?: 'JustReadFeedError';
   errorCodes: Array<JustReadFeedErrorCode>;
@@ -1418,7 +1424,8 @@ export type JustReadFeedSubscription = {
 
 export type JustReadFeedSuccess = {
   __typename?: 'JustReadFeedSuccess';
-  sections: Array<JustReadFeedSection>;
+  edges: Array<JustReadFeedEdge>;
+  pageInfo: PageInfo;
 };
 
 export type Label = {
@@ -4271,6 +4278,7 @@ export type ResolversTypes = {
   JoinGroupErrorCode: JoinGroupErrorCode;
   JoinGroupResult: ResolversTypes['JoinGroupError'] | ResolversTypes['JoinGroupSuccess'];
   JoinGroupSuccess: ResolverTypeWrapper<JoinGroupSuccess>;
+  JustReadFeedEdge: ResolverTypeWrapper<JustReadFeedEdge>;
   JustReadFeedError: ResolverTypeWrapper<JustReadFeedError>;
   JustReadFeedErrorCode: JustReadFeedErrorCode;
   JustReadFeedItem: ResolverTypeWrapper<JustReadFeedItem>;
@@ -4821,6 +4829,7 @@ export type ResolversParentTypes = {
   JoinGroupError: JoinGroupError;
   JoinGroupResult: ResolversParentTypes['JoinGroupError'] | ResolversParentTypes['JoinGroupSuccess'];
   JoinGroupSuccess: JoinGroupSuccess;
+  JustReadFeedEdge: JustReadFeedEdge;
   JustReadFeedError: JustReadFeedError;
   JustReadFeedItem: JustReadFeedItem;
   JustReadFeedResult: ResolversParentTypes['JustReadFeedError'] | ResolversParentTypes['JustReadFeedSuccess'];
@@ -6073,6 +6082,12 @@ export type JoinGroupSuccessResolvers<ContextType = ResolverContext, ParentType
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type JustReadFeedEdgeResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedEdge'] = ResolversParentTypes['JustReadFeedEdge']> = {
+  cursor?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  node?: Resolver<ResolversTypes['JustReadFeedSection'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type JustReadFeedErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedError'] = ResolversParentTypes['JustReadFeedError']> = {
   errorCodes?: Resolver<Array<ResolversTypes['JustReadFeedErrorCode']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -6123,7 +6138,8 @@ export type JustReadFeedSubscriptionResolvers<ContextType = ResolverContext, Par
 };
 
 export type JustReadFeedSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSuccess'] = ResolversParentTypes['JustReadFeedSuccess']> = {
-  sections?: Resolver<Array<ResolversTypes['JustReadFeedSection']>, ParentType, ContextType>;
+  edges?: Resolver<Array<ResolversTypes['JustReadFeedEdge']>, ParentType, ContextType>;
+  pageInfo?: Resolver<ResolversTypes['PageInfo'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
@@ -7650,6 +7666,7 @@ export type Resolvers<ContextType = ResolverContext> = {
   JoinGroupError?: JoinGroupErrorResolvers<ContextType>;
   JoinGroupResult?: JoinGroupResultResolvers<ContextType>;
   JoinGroupSuccess?: JoinGroupSuccessResolvers<ContextType>;
+  JustReadFeedEdge?: JustReadFeedEdgeResolvers<ContextType>;
   JustReadFeedError?: JustReadFeedErrorResolvers<ContextType>;
   JustReadFeedItem?: JustReadFeedItemResolvers<ContextType>;
   JustReadFeedResult?: JustReadFeedResultResolvers<ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 8899e679a..6a31ae89e 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1228,6 +1228,11 @@ type JoinGroupSuccess {
   group: RecommendationGroup!
 }
 
+type JustReadFeedEdge {
+  cursor: String!
+  node: JustReadFeedSection!
+}
+
 type JustReadFeedError {
   errorCodes: [JustReadFeedErrorCode!]!
 }
@@ -1278,7 +1283,8 @@ type JustReadFeedSubscription {
 }
 
 type JustReadFeedSuccess {
-  sections: [JustReadFeedSection!]!
+  edges: [JustReadFeedEdge!]!
+  pageInfo: PageInfo!
 }
 
 type Label {
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 30d5bf6ff..ffc316e29 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -2,11 +2,7 @@ import languages from '@cospired/i18n-iso-languages'
 import { LibraryItem } from '../entity/library_item'
 import { PublicItem } from '../entity/public_item'
 import { User } from '../entity/user'
-import {
-  JustReadFeedItem,
-  JustReadFeedSection,
-  JustReadFeedSuccess,
-} from '../generated/graphql'
+import { JustReadFeedSection } from '../generated/graphql'
 import { redisDataSource } from '../redis_data_source'
 import { findUnseenPublicItems } from '../services/just_read_feed'
 import { searchLibraryItems } from '../services/library_item'
@@ -38,7 +34,7 @@ interface ScoreApiRequestBody {
   item_features: Record<string, Feature> // item_id -> feature
 }
 
-interface Candidate {
+interface FeedItem {
   id: string
   title: string
   url: string
@@ -50,8 +46,13 @@ interface Candidate {
   date: Date
   topic?: string
   wordCount?: number
-  sourceIcon?: string
+  siteIcon?: string
   siteName?: string
+  saveCount?: number
+  likeCount?: number
+  broadcastCount?: number
+  folder?: string
+  subscriptionType: string
   subscription: {
     id: string
     name: string
@@ -62,10 +63,7 @@ interface Candidate {
 const lanaugeToCode = (language: string): string =>
   languages.getAlpha2Code(language, 'en') || 'en'
 
-const libraryItemToFeedItem = (
-  user: User,
-  item: LibraryItem
-): JustReadFeedItem => ({
+const libraryItemToCandidate = (user: User, item: LibraryItem): FeedItem => ({
   id: item.id,
   title: item.title,
   url: item.originalUrl,
@@ -77,8 +75,10 @@ const libraryItemToFeedItem = (
   date: item.createdAt,
   topic: item.topic,
   wordCount: item.wordCount || undefined,
-  sourceIcon: user.profile.pictureUrl || undefined,
   siteName: item.siteName || undefined,
+  siteIcon: item.siteIcon || undefined,
+  folder: item.folder,
+  subscriptionType: 'library',
   subscription: {
     id: user.id,
     name: user.name,
@@ -86,7 +86,7 @@ const libraryItemToFeedItem = (
   },
 })
 
-const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
+const publicItemToCandidate = (item: PublicItem): FeedItem => ({
   id: item.id,
   title: item.title,
   url: item.url,
@@ -97,19 +97,24 @@ const publicItemToFeedItem = (item: PublicItem): JustReadFeedItem => ({
   dir: item.dir || 'ltr',
   date: item.createdAt,
   topic: item.topic,
-  wordCount: item.wordCount || undefined,
+  wordCount: item.wordCount,
   siteName: item.siteName,
-  updatedAt: item.updatedAt,
   saveCount: item.stats.saveCount,
   likeCount: item.stats.likeCount,
   broadcastCount: item.stats.broadcastCount,
+  siteIcon: item.siteIcon,
+  // TODO
+  subscriptionType: 'public',
+  subscription: {
+    id: item.id,
+    name: item.siteName || '',
+    icon: item.siteIcon,
+  },
 })
 
 type ScoreApiResponse = Record<string, number> // item_id -> score
 
-const selectCandidates = async (
-  user: User
-): Promise<Array<JustReadFeedItem>> => {
+const selectCandidates = async (user: User): Promise<Array<FeedItem>> => {
   const userId = user.id
   // get last 100 library items saved and not seen by user
   const libraryItems = await searchLibraryItems(
@@ -123,51 +128,59 @@ const selectCandidates = async (
 
   logger.info(`Found ${libraryItems.length} library items`)
 
-  // map library items to candidates
-  const privateCandidates: Array<JustReadFeedItem> = libraryItems.map(
-    (libraryItem) => libraryItemToFeedItem(user, libraryItem)
-  )
+  // map library items to candidates and limit to 70
+  const privateCandidates: Array<FeedItem> = libraryItems
+    .map((libraryItem) => libraryItemToCandidate(user, libraryItem))
+    .slice(0, 70)
+  const privateCandidatesSize = privateCandidates.length
 
-  logger.info(`Found ${privateCandidates.length} private candidates`)
+  logger.info(`Found ${privateCandidatesSize} private candidates`)
 
-  // get candidates from public inventory
+  // get 100 items not seen by the user from public inventory
   const publicItems = await findUnseenPublicItems(userId, {
     limit: 100,
   })
 
   logger.info(`Found ${publicItems.length} public items`)
 
-  // map public items to candidates
-  const publicCandidates: Array<JustReadFeedItem> =
-    publicItems.map(publicItemToFeedItem)
+  // map public items to candidates and limit to the remaining vacancies
+  const publicCandidates: Array<FeedItem> = publicItems
+    .map(publicItemToCandidate)
+    .slice(0, 100 - privateCandidatesSize)
 
-  logger.info(`Found ${publicCandidates.length} public candidates`)
+  const publicCandidatesSize = publicCandidates.length
+  logger.info(`Found ${publicCandidatesSize} public candidates`)
 
-  // combine candidates
+  // return 100 candidates and 70 from private and 30 from public
   return [...privateCandidates, ...publicCandidates]
 }
 
-const rankFeedItems = async (
+const rankCandidates = async (
   userId: string,
-  feedItems: Array<JustReadFeedItem>
-): Promise<Array<JustReadFeedItem>> => {
-  if (feedItems.length <= 10) {
+  candidates: Array<FeedItem>
+): Promise<Array<FeedItem>> => {
+  if (candidates.length <= 10) {
     // no need to rank if there are less than 10 candidates
-    return feedItems
+    return candidates
   }
 
   // TODO: get score of candidates
   const API_URL = 'http://127.0.0.1:5000/predictions'
   const requestBody: ScoreApiRequestBody = {
     user_id: userId,
-    item_features: feedItems.reduce((acc, item) => {
+    item_features: candidates.reduce((acc, item) => {
       acc[item.id] = {
         title: item.title,
         has_thumbnail: !!item.thumbnail,
-        has_site_icon: !!item.sourceIcon,
+        has_site_icon: !!item.subscription.icon,
         saved_at: item.date,
         site: item.siteName,
         language: item.languageCode,
+        directionality: item.dir,
+        folder: item.subscription.name,
+        subscription_type: item.subscriptionType,
+        author: item.author,
+        word_count: item.wordCount,
       } as Feature
       return acc
     }, {} as Record<string, Feature>),
@@ -187,31 +200,25 @@ const rankFeedItems = async (
 
   const scores = (await response.json()) as ScoreApiResponse
 
-  // // fake scores
-  // const scores: Array<FeedItemScore> = feedItems.map((item) => ({
-  //   id: item.id,
-  //   score: Math.random(),
-  // }))
-
   // rank candidates by score in ascending order
-  feedItems.sort((a, b) => {
+  candidates.sort((a, b) => {
     const scoreA = scores[a.id] || 0
     const scoreB = scores[b.id] || 0
 
     return scoreA - scoreB
   })
 
-  return Promise.resolve(feedItems)
+  return candidates
 }
 
 const redisKey = (userId: string) => `just-read-feed:${userId}`
 const MAX_FEED_ITEMS = 500
 
-export const getJustReadFeed = async (
+export const getJustReadFeedSections = async (
   userId: string,
-  first = 10,
-  after?: string
-): Promise<JustReadFeedSuccess> => {
+  limit: number,
+  minScore?: number
+): Promise<Array<{ member: JustReadFeedSection; score: number }>> => {
   const redisClient = redisDataSource.redisClient
   if (!redisClient) {
     throw new Error('Redis client not available')
@@ -219,11 +226,28 @@ export const getJustReadFeed = async (
 
   const key = redisKey(userId)
 
-  const results = await redisClient.zrevrange(key, 0, first - 1, 'WITHSCORES')
+  // get feed items from redis sorted set in descending order
+  // with score greater than minScore
+  // limit to the first `limit` items
+  // response is an array of [member1, score1, member2, score2, ...]
+  const results = await redisClient.zrevrangebyscore(
+    key,
+    '+inf',
+    minScore || '-inf',
+    'WITHSCORES',
+    'LIMIT',
+    0,
+    limit
+  )
 
-  const sections = JSON.parse(results.join(',')) as JustReadFeedSection[]
+  const sections = []
+  for (let i = 0; i < results.length; i += 2) {
+    const member = JSON.parse(results[i]) as JustReadFeedSection
+    const score = Number(results[i + 1])
+    sections.push({ member, score })
+  }
 
-  return { sections }
+  return sections
 }
 
 const appendSectionsToFeed = async (
@@ -240,9 +264,9 @@ const appendSectionsToFeed = async (
   // store candidates in redis sorted set
   const pipeline = redisClient.pipeline()
 
-  const scoreMembers = sections.flatMap((item) => [
+  const scoreMembers = sections.flatMap((section) => [
     Date.now() + 86_400_000, // sections expire in 24 hours
-    JSON.stringify(item),
+    JSON.stringify(section),
   ])
   // add section to the sorted set
   pipeline.zadd(key, ...scoreMembers)
@@ -255,6 +279,92 @@ const appendSectionsToFeed = async (
   await pipeline.exec()
 }
 
+const mixFeedItems = (
+  rankedFeedItems: Array<FeedItem>
+): Array<JustReadFeedSection> => {
+  // find the median word count
+  const wordCounts = rankedFeedItems.map((item) => item.wordCount || 0)
+  wordCounts.sort()
+  const medianWordCount = wordCounts[Math.floor(wordCounts.length / 2)]
+  // separate items into two groups based on word count
+  const shortItems: Array<FeedItem> = []
+  const longItems: Array<FeedItem> = []
+  for (const item of rankedFeedItems) {
+    if (item.wordCount && item.wordCount < medianWordCount) {
+      shortItems.push(item)
+    } else {
+      longItems.push(item)
+    }
+  }
+  // initialize empty batches
+  const batches = [[]]
+
+  const checkConstraints = (batch: Array<FeedItem>, item: FeedItem) => {
+    const titleCount = batch.filter((i) => i.title === item.title).length
+    const authorCount = batch.filter((i) => i.author === item.author).length
+    const siteCount = batch.filter((i) => i.siteName === item.siteName).length
+    const subscriptionCount = batch.filter(
+      (i) => i.subscription.name === item.subscription.name
+    ).length
+
+    return (
+      titleCount < 1 &&
+      authorCount < 2 &&
+      siteCount < 2 &&
+      subscriptionCount < 2
+    )
+  }
+
+  const distributeItems = (
+    items: Array<FeedItem>,
+    batches: Array<Array<FeedItem>>
+  ) => {
+    for (const item of items) {
+      let added = false
+      for (const batch of batches) {
+        if (batch.length < 5 && checkConstraints(batch, item)) {
+          batch.push(item)
+          added = true
+          break
+        }
+      }
+
+      if (!added) {
+        for (const batch of batches) {
+          if (batch.length < 5) {
+            batch.push(item)
+            break
+          }
+        }
+      }
+    }
+  }
+
+  // distribute longer items first and then shorter items
+  distributeItems(longItems, batches)
+  distributeItems(shortItems, batches)
+
+  // convert batches to sections
+  const sections = []
+  for (const batch of batches) {
+    // create a section for each long item
+    for (let i = 0; i < 5; i++) {
+      const section: JustReadFeedSection = {
+        items: [batch[i]],
+        layout: 'long',
+      }
+      sections.push(section)
+    }
+    // create a section for short items
+    sections.push({
+      items: batch.slice(5),
+      layout: 'quick links',
+    })
+  }
+
+  return sections
+}
+
 export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
   const { userId } = data
   const user = await findActiveUser(userId)
@@ -265,23 +375,25 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
 
   logger.info(`Updating just read feed for user ${userId}`)
 
-  const feedItems = await selectCandidates(user)
-  logger.info(`Found ${feedItems.length} candidates`)
+  const candidates = await selectCandidates(user)
+  logger.info(`Found ${candidates.length} candidates`)
 
-  // TODO: integrity check on candidates?
+  // TODO: integrity check on candidates
 
-  logger.info('Ranking feed items')
-  const rankedFeedItems = await rankFeedItems(userId, feedItems)
-  if (rankedFeedItems.length === 0) {
-    logger.info('No feed items to append')
+  logger.info('Ranking candidates')
+  const rankedCandidates = await rankCandidates(userId, candidates)
+  if (rankedCandidates.length === 0) {
+    logger.info('No candidates found')
     return
   }
 
-  logger.info('Filtering feed items')
-  // TODO: filtering
-  // get top 100 ranked feed items
-  const filteredFeedItems = rankedFeedItems.slice(0, 100)
+  // TODO: filter candidates
+
+  logger.info('Mix feed items to create sections')
+  const rankedSections = mixFeedItems(rankedCandidates)
+  logger.info(`Created ${rankedSections.length} sections`)
 
   logger.info('Appending feed items to feed')
-  await appendSectionsToFeed(userId, filteredFeedItems)
+  await appendSectionsToFeed(userId, rankedSections)
+  logger.info('Feed updated for user', { userId })
 }
diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/just_read_feed/index.ts
index 9b03219ad..b2fe5a8f3 100644
--- a/packages/api/src/resolvers/just_read_feed/index.ts
+++ b/packages/api/src/resolvers/just_read_feed/index.ts
@@ -1,20 +1,28 @@
 import {
   JustReadFeedError,
   JustReadFeedErrorCode,
-  JustReadFeedSuccess
+  JustReadFeedSuccess,
+  QueryJustReadFeedArgs,
 } from '../../generated/graphql'
-import { getJustReadFeed } from '../../jobs/update_just_read_feed'
+import { getJustReadFeedSections } from '../../jobs/update_just_read_feed'
 import { enqueueUpdateJustReadFeed } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 
+// This resolver is used to fetch the just read feed for the user.
+// when the feed is empty, it enqueues a job to update the feed.
+// when client tries to fetch more then the feed has, it enqueues a job to update the feed.
 export const justReadFeedResolver = authorized<
   JustReadFeedSuccess,
-  JustReadFeedError
+  JustReadFeedError,
+  QueryJustReadFeedArgs
 >(async (_, { first, after }, { uid, log }) => {
-  const feed = await getJustReadFeed(uid, first, after)
-  log.info('Just read feed fetched')
+  const limit = first || 10
+  const minScore = after ? Number(after) : 0
 
-  if (feed.sections.length === 0) {
+  const sections = await getJustReadFeedSections(uid, limit, minScore)
+  log.info('Just read feed sections fetched')
+
+  if (sections.length === 0) {
     await enqueueUpdateJustReadFeed({
       userId: uid,
     })
@@ -26,5 +34,16 @@ export const justReadFeedResolver = authorized<
     }
   }
 
-  return feed
+  const edges = sections.map((section) => ({
+    cursor: section.score.toString(),
+    node: section.member,
+  }))
+
+  return {
+    edges,
+    pageInfo: {
+      hasPreviousPage: true, // there is always a previous page for new items
+      hasNextPage: true, // there is always a next page for old items
+    },
+  }
 })
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 2c8d7e3aa..3a7c0bd4c 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3138,8 +3138,14 @@ const schema = gql`
     thumbnail: String
   }
 
+  type JustReadFeedEdge {
+    cursor: String!
+    node: JustReadFeedSection!
+  }
+
   type JustReadFeedSuccess {
-    sections: [JustReadFeedSection!]!
+    edges: [JustReadFeedEdge!]!
+    pageInfo: PageInfo!
   }
 
   enum JustReadFeedErrorCode {

From 67dac9ba315d8fa6703e1690ed7fbdf7717ce520 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sun, 26 May 2024 14:41:38 +0800
Subject: [PATCH 13/33] add public item source

---
 packages/api/src/entity/public_item.ts        |  7 ++++
 packages/api/src/entity/public_item_source.ts | 40 +++++++++++++++++++
 .../api/src/jobs/update_just_read_feed.ts     |  9 ++---
 packages/api/src/services/just_read_feed.ts   |  5 +++
 .../db/migrations/0177.do.public_item.sql     | 13 +++---
 5 files changed, 63 insertions(+), 11 deletions(-)
 create mode 100644 packages/api/src/entity/public_item_source.ts

diff --git a/packages/api/src/entity/public_item.ts b/packages/api/src/entity/public_item.ts
index e2843a399..78a174e31 100644
--- a/packages/api/src/entity/public_item.ts
+++ b/packages/api/src/entity/public_item.ts
@@ -2,10 +2,13 @@ import {
   Column,
   CreateDateColumn,
   Entity,
+  JoinColumn,
+  ManyToOne,
   OneToOne,
   PrimaryGeneratedColumn,
   UpdateDateColumn,
 } from 'typeorm'
+import { PublicItemSource } from './public_item_source'
 import { PublicItemStats } from './public_item_stats'
 
 @Entity({ name: 'public_item' })
@@ -16,6 +19,10 @@ export class PublicItem {
   @OneToOne(() => PublicItemStats)
   stats!: PublicItemStats
 
+  @ManyToOne(() => PublicItemSource)
+  @JoinColumn({ name: 'source_id' })
+  source!: PublicItemSource
+
   @Column('text')
   siteIcon?: string
 
diff --git a/packages/api/src/entity/public_item_source.ts b/packages/api/src/entity/public_item_source.ts
new file mode 100644
index 000000000..a79a6ec19
--- /dev/null
+++ b/packages/api/src/entity/public_item_source.ts
@@ -0,0 +1,40 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  PrimaryGeneratedColumn,
+  UpdateDateColumn,
+} from 'typeorm'
+
+@Entity({ name: 'public_item_source' })
+export class PublicItemSource {
+  @PrimaryGeneratedColumn('uuid')
+  id!: string
+
+  @Column('text')
+  type!: string
+
+  @Column('text')
+  name!: string
+
+  @Column('text')
+  url?: string
+
+  @Column('boolean')
+  approved!: boolean
+
+  @Column('text')
+  icon?: string
+
+  @Column('text')
+  topics?: string[]
+
+  @Column('text')
+  languageCodes?: string[]
+
+  @CreateDateColumn()
+  createdAt!: Date
+
+  @UpdateDateColumn()
+  updatedAt!: Date
+}
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index ffc316e29..3f75ae8ad 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -103,12 +103,11 @@ const publicItemToCandidate = (item: PublicItem): FeedItem => ({
   likeCount: item.stats.likeCount,
   broadcastCount: item.stats.broadcastCount,
   siteIcon: item.siteIcon,
-  // TODO
-  subscriptionType: 'public',
+  subscriptionType: item.type,
   subscription: {
-    id: item.id,
-    name: item.siteName || '',
-    icon: item.siteIcon,
+    id: item.source.id,
+    name: item.source.name,
+    icon: item.source.icon,
   },
 })
 
diff --git a/packages/api/src/services/just_read_feed.ts b/packages/api/src/services/just_read_feed.ts
index 7d8a18103..895192062 100644
--- a/packages/api/src/services/just_read_feed.ts
+++ b/packages/api/src/services/just_read_feed.ts
@@ -23,6 +23,11 @@ export const findUnseenPublicItems = async (
           'stats',
           'stats.public_item_id = public_item.id'
         )
+        .innerJoin(
+          'public_item_source',
+          'source',
+          'source.id = public_item.source_id'
+        )
         .where('interaction.user_id = :userId', { userId })
         .andWhere('interaction.seen_at IS NULL')
         .orderBy('public_item.createdAt', 'DESC')
diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index 32a618660..7e45df56f 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -7,10 +7,11 @@ BEGIN;
 CREATE TABLE omnivore.public_item_source (
     id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
     name TEXT NOT NULL,
-    topics TEXT[] NOT NULL,
-    thumbnail TEXT,
-    url TEXT NOT NULL,
-    language_codes TEXT[] NOT NULL,
+    type TEXT NOT NULL, -- public feeds, newsletters, or user recommended
+    topics TEXT[],
+    icon TEXT,
+    url TEXT,
+    language_codes TEXT[],
     approved BOOLEAN NOT NULL DEFAULT FALSE,
     created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
@@ -22,8 +23,8 @@ GRANT SELECT ON omnivore.public_item_source TO omnivore_user;
 
 CREATE TABLE omnivore.public_item (
 	id uuid PRIMARY KEY DEFAULT uuid_generate_v1mc(),
-    source_name TEXT NOT NULL,
-    source_icon TEXT,
+    source_id uuid NOT NULL REFERENCES omnivore.public_item_source(id) ON DELETE CASCADE,
+    site_icon TEXT,
     type TEXT NOT NULL, -- public feeds, newsletters, or user recommended
     title TEXT NOT NULL,
     url TEXT NOT NULL,

From 25a984dc6a9f9467cd0b3b1020542602da84eb8d Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Sun, 26 May 2024 14:48:30 +0800
Subject: [PATCH 14/33] check for existing job before enqueueing

---
 .../api/src/resolvers/just_read_feed/index.ts     | 15 ++++++++++++++-
 packages/api/src/utils/createTask.ts              |  5 ++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/just_read_feed/index.ts
index b2fe5a8f3..e6ff119a6 100644
--- a/packages/api/src/resolvers/just_read_feed/index.ts
+++ b/packages/api/src/resolvers/just_read_feed/index.ts
@@ -5,7 +5,11 @@ import {
   QueryJustReadFeedArgs,
 } from '../../generated/graphql'
 import { getJustReadFeedSections } from '../../jobs/update_just_read_feed'
-import { enqueueUpdateJustReadFeed } from '../../utils/createTask'
+import { getJob } from '../../queue-processor'
+import {
+  enqueueUpdateJustReadFeed,
+  updateJustReadFeedJobId,
+} from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 
 // This resolver is used to fetch the just read feed for the user.
@@ -23,6 +27,15 @@ export const justReadFeedResolver = authorized<
   log.info('Just read feed sections fetched')
 
   if (sections.length === 0) {
+    const existingJob = await getJob(updateJustReadFeedJobId(uid))
+    if (existingJob) {
+      log.info('Just read feed update job already enqueued')
+
+      return {
+        errorCodes: [JustReadFeedErrorCode.Pending],
+      }
+    }
+
     await enqueueUpdateJustReadFeed({
       userId: uid,
     })
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 5903d8193..3f4f12061 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -986,6 +986,9 @@ export const enqueueBulkUploadContentJob = async (
   return queue.addBulk(jobs)
 }
 
+export const updateJustReadFeedJobId = (userId: string) =>
+  `${UPDATE_JUST_READ_FEED_JOB}_${userId}_${JOB_VERSION}`
+
 export const enqueueUpdateJustReadFeed = async (
   data: UpdateJustReadFeedJobData
 ) => {
@@ -995,7 +998,7 @@ export const enqueueUpdateJustReadFeed = async (
   }
 
   return queue.add(UPDATE_JUST_READ_FEED_JOB, data, {
-    jobId: `${UPDATE_JUST_READ_FEED_JOB}_${data.userId}_${JOB_VERSION}`,
+    jobId: updateJustReadFeedJobId(data.userId),
     removeOnComplete: true,
     removeOnFail: true,
     priority: getJobPriority(UPDATE_JUST_READ_FEED_JOB),

From 80841bf9af3fb1060b2adee40f5f9fb5d758caf2 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 11:11:05 +0800
Subject: [PATCH 15/33] add score to library_item table

---
 packages/db/migrations/0177.do.public_item.sql   | 3 ++-
 packages/db/migrations/0177.undo.public_item.sql | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index 7e45df56f..c4b465845 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -83,6 +83,7 @@ GRANT SELECT, INSERT, UPDATE ON omnivore.public_item_interactions TO omnivore_us
 ALTER TABLE omnivore.library_item 
     ADD COLUMN seen_at timestamptz,
     ADD COLUMN digested_at timestamptz,
-    ADD COLUMN topic TEXT;
+    ADD COLUMN topic TEXT,
+    ADD COLUMN score FLOAT;
 
 COMMIT;
diff --git a/packages/db/migrations/0177.undo.public_item.sql b/packages/db/migrations/0177.undo.public_item.sql
index a58f38333..314dcccf2 100755
--- a/packages/db/migrations/0177.undo.public_item.sql
+++ b/packages/db/migrations/0177.undo.public_item.sql
@@ -12,6 +12,7 @@ DROP TABLE omnivore.public_item_source;
 ALTER TABLE omnivore.library_item 
     DROP COLUMN seen_at,
     DROP COLUMN digested_at,
-    DROP COLUMN topic;
+    DROP COLUMN topic,
+    DROP COLUMN score;
 
 COMMIT;

From f6424c3bd28572e3acc534fa513638928529fa2a Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 11:12:07 +0800
Subject: [PATCH 16/33] add score to library_item entity

---
 packages/api/src/entity/library_item.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts
index 2950ff9a6..8168b8548 100644
--- a/packages/api/src/entity/library_item.ts
+++ b/packages/api/src/entity/library_item.ts
@@ -213,4 +213,7 @@ export class LibraryItem {
 
   @Column('timestamptz')
   digestedAt?: Date
+
+  @Column('float')
+  score?: number
 }

From 3cb93283b0e4a14f5128fa39e33f5e15377dab25 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 11:37:59 +0800
Subject: [PATCH 17/33] add score library item job

---
 packages/api/src/jobs/score_library_item.ts   | 80 +++++++++++++++++++
 .../api/src/jobs/update_just_read_feed.ts     | 45 +----------
 packages/api/src/services/score.ts            | 41 ++++++++++
 packages/api/src/utils/helpers.ts             |  5 ++
 4 files changed, 130 insertions(+), 41 deletions(-)
 create mode 100644 packages/api/src/jobs/score_library_item.ts
 create mode 100644 packages/api/src/services/score.ts

diff --git a/packages/api/src/jobs/score_library_item.ts b/packages/api/src/jobs/score_library_item.ts
new file mode 100644
index 000000000..5708316c6
--- /dev/null
+++ b/packages/api/src/jobs/score_library_item.ts
@@ -0,0 +1,80 @@
+import {
+  findLibraryItemById,
+  updateLibraryItem,
+} from '../services/library_item'
+import { Feature, getScores } from '../services/score'
+import { lanaugeToCode } from '../utils/helpers'
+import { logger } from '../utils/logger'
+
+export const SCORE_LIBRARY_ITEM_JOB = 'SCORE_LIBRARY_ITEM_JOB'
+
+export interface ScoreLibraryItemJobData {
+  userId: string
+  libraryItemId: string
+}
+
+export const scoreLibraryItem = async (
+  data: ScoreLibraryItemJobData
+): Promise<void> => {
+  logger.info('Scoring library item', data)
+
+  const { userId, libraryItemId } = data
+
+  const libraryItem = await findLibraryItemById(libraryItemId, userId, {
+    select: [
+      'id',
+      'title',
+      'thumbnail',
+      'siteIcon',
+      'savedAt',
+      'siteName',
+      'directionality',
+      'folder',
+      'author',
+      'itemLanguage',
+      'wordCount',
+    ],
+  })
+  if (!libraryItem) {
+    logger.error('Library item not found', data)
+    return
+  }
+
+  const itemFeatures = {
+    [libraryItem.id]: {
+      title: libraryItem.title,
+      has_thumbnail: !!libraryItem.thumbnail,
+      has_site_icon: !!libraryItem.siteIcon,
+      saved_at: libraryItem.savedAt,
+      site: libraryItem.siteName,
+      directionality: libraryItem.directionality,
+      folder: libraryItem.folder,
+      subscription_type: 'library',
+      author: libraryItem.author,
+      language: lanaugeToCode(libraryItem.itemLanguage || 'English'),
+      word_count: libraryItem.wordCount,
+    } as Feature,
+  }
+
+  const scores = await getScores({
+    user_id: userId,
+    item_features: itemFeatures,
+  })
+
+  logger.info('Scores', scores)
+  const score = scores[libraryItem.id]
+  if (!score) {
+    logger.error('Failed to score library item', data)
+    throw new Error('Failed to score library item')
+  }
+
+  await updateLibraryItem(
+    libraryItem.id,
+    {
+      score,
+    },
+    userId,
+    undefined,
+    true
+  )
+}
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 3f75ae8ad..85eca3d68 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -1,4 +1,3 @@
-import languages from '@cospired/i18n-iso-languages'
 import { LibraryItem } from '../entity/library_item'
 import { PublicItem } from '../entity/public_item'
 import { User } from '../entity/user'
@@ -6,7 +5,9 @@ import { JustReadFeedSection } from '../generated/graphql'
 import { redisDataSource } from '../redis_data_source'
 import { findUnseenPublicItems } from '../services/just_read_feed'
 import { searchLibraryItems } from '../services/library_item'
+import { Feature, getScores } from '../services/score'
 import { findActiveUser } from '../services/user'
+import { lanaugeToCode } from '../utils/helpers'
 import { logger } from '../utils/logger'
 
 export const UPDATE_JUST_READ_FEED_JOB = 'UPDATE_JUST_READ_FEED_JOB'
@@ -15,25 +16,6 @@ export interface UpdateJustReadFeedJobData {
   userId: string
 }
 
-interface Feature {
-  title: string
-  has_thumbnail: boolean
-  has_site_icon: boolean
-  saved_at: Date
-  site?: string
-  language?: string
-  author?: string
-  directionality: string
-  word_count?: number
-  subscription_type: string
-  folder: string
-}
-
-interface ScoreApiRequestBody {
-  user_id: string
-  item_features: Record<string, Feature> // item_id -> feature
-}
-
 interface FeedItem {
   id: string
   title: string
@@ -60,9 +42,6 @@ interface FeedItem {
   }
 }
 
-const lanaugeToCode = (language: string): string =>
-  languages.getAlpha2Code(language, 'en') || 'en'
-
 const libraryItemToCandidate = (user: User, item: LibraryItem): FeedItem => ({
   id: item.id,
   title: item.title,
@@ -111,8 +90,6 @@ const publicItemToCandidate = (item: PublicItem): FeedItem => ({
   },
 })
 
-type ScoreApiResponse = Record<string, number> // item_id -> score
-
 const selectCandidates = async (user: User): Promise<Array<FeedItem>> => {
   const userId = user.id
   // get last 100 library items saved and not seen by user
@@ -163,9 +140,7 @@ const rankCandidates = async (
     return candidates
   }
 
-  // TODO: get score of candidates
-  const API_URL = 'http://127.0.0.1:5000/predictions'
-  const requestBody: ScoreApiRequestBody = {
+  const data = {
     user_id: userId,
     item_features: candidates.reduce((acc, item) => {
       acc[item.id] = {
@@ -185,19 +160,7 @@ const rankCandidates = async (
     }, {} as Record<string, Feature>),
   }
 
-  const response = await fetch(API_URL, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify(requestBody),
-  })
-
-  if (!response.ok) {
-    throw new Error(`Failed to score candidates: ${response.statusText}`)
-  }
-
-  const scores = (await response.json()) as ScoreApiResponse
+  const scores = await getScores(data)
 
   // rank candidates by score in ascending order
   candidates.sort((a, b) => {
diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
new file mode 100644
index 000000000..8e9d2785c
--- /dev/null
+++ b/packages/api/src/services/score.ts
@@ -0,0 +1,41 @@
+export interface Feature {
+  title: string
+  has_thumbnail: boolean
+  has_site_icon: boolean
+  saved_at: Date
+  site?: string
+  language?: string
+  author?: string
+  directionality: string
+  word_count?: number
+  subscription_type: string
+  folder: string
+}
+
+export interface ScoreApiRequestBody {
+  user_id: string
+  item_features: Record<string, Feature> // item_id -> feature
+}
+
+export type ScoreApiResponse = Record<string, number> // item_id -> score
+
+export const getScores = async (
+  data: ScoreApiRequestBody
+): Promise<ScoreApiResponse> => {
+  const API_URL = 'http://127.0.0.1:5000/predictions'
+
+  const response = await fetch(API_URL, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(data),
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to score candidates: ${response.statusText}`)
+  }
+
+  const scores = (await response.json()) as ScoreApiResponse
+  return scores
+}
diff --git a/packages/api/src/utils/helpers.ts b/packages/api/src/utils/helpers.ts
index 01aaf9e94..30a821a5e 100644
--- a/packages/api/src/utils/helpers.ts
+++ b/packages/api/src/utils/helpers.ts
@@ -1,4 +1,5 @@
 /* eslint-disable @typescript-eslint/no-unsafe-assignment */
+import languages from '@cospired/i18n-iso-languages'
 import crypto from 'crypto'
 import Redis from 'ioredis'
 import normalizeUrl from 'normalize-url'
@@ -31,6 +32,7 @@ import { validateUrl } from '../services/create_page_save_request'
 import { updateLibraryItem } from '../services/library_item'
 import { Merge } from '../util'
 import { logger } from './logger'
+
 interface InputObject {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   [key: string]: any
@@ -423,3 +425,6 @@ export const getClientFromUserAgent = (userAgent: string): string => {
 
   return 'other'
 }
+
+export const lanaugeToCode = (language: string): string =>
+  languages.getAlpha2Code(language, 'en') || 'en'

From 41b9fef3163566a51811badfac6149bb5e467dc9 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 11:45:51 +0800
Subject: [PATCH 18/33] score library item after saved

---
 packages/api/src/pubsub.ts           |  6 ++++++
 packages/api/src/queue-processor.ts  |  6 ++++++
 packages/api/src/utils/createTask.ts | 22 ++++++++++++++++++++++
 3 files changed, 34 insertions(+)

diff --git a/packages/api/src/pubsub.ts b/packages/api/src/pubsub.ts
index fea9139b7..3e6fea7fd 100644
--- a/packages/api/src/pubsub.ts
+++ b/packages/api/src/pubsub.ts
@@ -5,6 +5,7 @@ import { env } from './env'
 import { ReportType } from './generated/graphql'
 import {
   enqueueProcessYouTubeVideo,
+  enqueueScoreJob,
   enqueueTriggerRuleJob,
 } from './utils/createTask'
 import { logger } from './utils/logger'
@@ -74,6 +75,11 @@ export const createPubSubClient = (): PubsubClient => {
             libraryItemId: data.id,
           })
         }
+
+        await enqueueScoreJob({
+          userId,
+          libraryItemId: data.id,
+        })
       }
     },
     entityUpdated: async <T extends EntityEvent>(
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index edb634e70..ea7a8fe16 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -48,6 +48,10 @@ import {
 import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
 import { refreshFeed } from './jobs/rss/refreshFeed'
 import { savePageJob } from './jobs/save_page'
+import {
+  scoreLibraryItem,
+  SCORE_LIBRARY_ITEM_JOB,
+} from './jobs/score_library_item'
 import {
   syncReadPositionsJob,
   SYNC_READ_POSITIONS_JOB_NAME,
@@ -191,6 +195,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           return uploadContentJob(job.data)
         case UPDATE_JUST_READ_FEED_JOB:
           return updateJustReadFeed(job.data)
+        case SCORE_LIBRARY_ITEM_JOB:
+          return scoreLibraryItem(job.data)
         default:
           logger.warning(`[queue-processor] unhandled job: ${job.name}`)
       }
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 3f4f12061..7b8342aad 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -45,6 +45,10 @@ import {
   REFRESH_ALL_FEEDS_JOB_NAME,
   REFRESH_FEED_JOB_NAME,
 } from '../jobs/rss/refreshAllFeeds'
+import {
+  ScoreLibraryItemJobData,
+  SCORE_LIBRARY_ITEM_JOB,
+} from '../jobs/score_library_item'
 import { SYNC_READ_POSITIONS_JOB_NAME } from '../jobs/sync_read_positions'
 import { TriggerRuleJobData, TRIGGER_RULE_JOB_NAME } from '../jobs/trigger_rule'
 import {
@@ -1006,4 +1010,22 @@ export const enqueueUpdateJustReadFeed = async (
   })
 }
 
+export const updateScoreJobId = (userId: string) =>
+  `${SCORE_LIBRARY_ITEM_JOB}_${userId}_${JOB_VERSION}`
+
+export const enqueueScoreJob = async (data: ScoreLibraryItemJobData) => {
+  const queue = await getBackendQueue()
+  if (!queue) {
+    return undefined
+  }
+
+  return queue.add(SCORE_LIBRARY_ITEM_JOB, data, {
+    jobId: updateScoreJobId(data.userId),
+    removeOnComplete: true,
+    removeOnFail: true,
+    priority: getJobPriority(SCORE_LIBRARY_ITEM_JOB),
+    attempts: 3,
+  })
+}
+
 export default createHttpTaskWithToken

From 8f0bebf38151b98b8e18bef970b627da97e429fc Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 11:55:44 +0800
Subject: [PATCH 19/33] use pre-calculated score from library item when ranking

---
 .../api/src/jobs/update_just_read_feed.ts     | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 85eca3d68..c3bf180e2 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -5,7 +5,7 @@ import { JustReadFeedSection } from '../generated/graphql'
 import { redisDataSource } from '../redis_data_source'
 import { findUnseenPublicItems } from '../services/just_read_feed'
 import { searchLibraryItems } from '../services/library_item'
-import { Feature, getScores } from '../services/score'
+import { Feature, getScores, ScoreApiResponse } from '../services/score'
 import { findActiveUser } from '../services/user'
 import { lanaugeToCode } from '../utils/helpers'
 import { logger } from '../utils/logger'
@@ -35,6 +35,7 @@ interface FeedItem {
   broadcastCount?: number
   folder?: string
   subscriptionType: string
+  score?: number
   subscription: {
     id: string
     name: string
@@ -58,6 +59,7 @@ const libraryItemToCandidate = (user: User, item: LibraryItem): FeedItem => ({
   siteIcon: item.siteIcon || undefined,
   folder: item.folder,
   subscriptionType: 'library',
+  score: item.score,
   subscription: {
     id: user.id,
     name: user.name,
@@ -140,9 +142,13 @@ const rankCandidates = async (
     return candidates
   }
 
+  const unscoredCandidates = candidates.filter(
+    (item) => item.score === undefined
+  )
+
   const data = {
     user_id: userId,
-    item_features: candidates.reduce((acc, item) => {
+    item_features: unscoredCandidates.reduce((acc, item) => {
       acc[item.id] = {
         title: item.title,
         has_thumbnail: !!item.thumbnail,
@@ -160,7 +166,14 @@ const rankCandidates = async (
     }, {} as Record<string, Feature>),
   }
 
-  const scores = await getScores(data)
+  const newScores = await getScores(data)
+  const preCalculatedScores = candidates
+    .filter((item) => item.score !== undefined)
+    .reduce((acc, item) => {
+      acc[item.id] = item.score as number
+      return acc
+    }, {} as ScoreApiResponse)
+  const scores = { ...preCalculatedScores, ...newScores }
 
   // rank candidates by score in ascending order
   candidates.sort((a, b) => {

From fff3b4b5237d6d624f6108d6304557b92c7d898c Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 11:58:51 +0800
Subject: [PATCH 20/33] add authorization to the score api client

---
 packages/api/src/services/score.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
index 8e9d2785c..c12069a46 100644
--- a/packages/api/src/services/score.ts
+++ b/packages/api/src/services/score.ts
@@ -23,11 +23,17 @@ export const getScores = async (
   data: ScoreApiRequestBody
 ): Promise<ScoreApiResponse> => {
   const API_URL = 'http://127.0.0.1:5000/predictions'
+  const token = process.env.SCORE_API_TOKEN
+
+  if (!token) {
+    throw new Error('No score API token found')
+  }
 
   const response = await fetch(API_URL, {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json',
+      Authorization: `Bearer ${token}`,
     },
     body: JSON.stringify(data),
   })

From 88c25844b44d2ebde4f11fef8b34fe3b7d45f60e Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 12:50:57 +0800
Subject: [PATCH 21/33] set priority = 10 for score job

---
 packages/api/src/utils/createTask.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 7b8342aad..2ba80e4be 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -104,6 +104,7 @@ export const getJobPriority = (jobName: string): number => {
     case `${REFRESH_FEED_JOB_NAME}_high`:
     case PROCESS_YOUTUBE_TRANSCRIPT_JOB_NAME:
     case UPLOAD_CONTENT_JOB:
+    case SCORE_LIBRARY_ITEM_JOB:
       return 10
     case `${REFRESH_FEED_JOB_NAME}_low`:
     case EXPORT_ITEM_JOB_NAME:

From 87ad71fbe9d425dd57e9ab75abb2d016854c5ef6 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 12:56:38 +0800
Subject: [PATCH 22/33] add published_at to the score api request

---
 packages/api/src/jobs/update_just_read_feed.ts | 4 ++++
 packages/api/src/services/score.ts             | 8 ++++++++
 2 files changed, 12 insertions(+)

diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index c3bf180e2..e2d5c5e38 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -36,6 +36,7 @@ interface FeedItem {
   folder?: string
   subscriptionType: string
   score?: number
+  publishedAt?: Date
   subscription: {
     id: string
     name: string
@@ -60,6 +61,7 @@ const libraryItemToCandidate = (user: User, item: LibraryItem): FeedItem => ({
   folder: item.folder,
   subscriptionType: 'library',
   score: item.score,
+  publishedAt: item.publishedAt || undefined,
   subscription: {
     id: user.id,
     name: user.name,
@@ -85,6 +87,7 @@ const publicItemToCandidate = (item: PublicItem): FeedItem => ({
   broadcastCount: item.stats.broadcastCount,
   siteIcon: item.siteIcon,
   subscriptionType: item.type,
+  publishedAt: item.publishedAt,
   subscription: {
     id: item.source.id,
     name: item.source.name,
@@ -161,6 +164,7 @@ const rankCandidates = async (
         subscription_type: item.subscriptionType,
         author: item.author,
         word_count: item.wordCount,
+        published_at: item.publishedAt,
       } as Feature
       return acc
     }, {} as Record<string, Feature>),
diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
index c12069a46..ed1820efc 100644
--- a/packages/api/src/services/score.ts
+++ b/packages/api/src/services/score.ts
@@ -10,6 +10,7 @@ export interface Feature {
   word_count?: number
   subscription_type: string
   folder: string
+  published_at?: Date
 }
 
 export interface ScoreApiRequestBody {
@@ -44,4 +45,11 @@ export const getScores = async (
 
   const scores = (await response.json()) as ScoreApiResponse
   return scores
+
+  // fake random scores
+  // const scores: ScoreApiResponse = {}
+  // for (const itemId of Object.keys(data.item_features)) {
+  //   scores[itemId] = Math.random()
+  // }
+  // return Promise.resolve(scores)
 }

From f603c84b270c17f6820c7d4e7fe804e8fb13f7c1 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 17:38:36 +0800
Subject: [PATCH 23/33] store only feed item id and type in redis and fetch the
 details from DB when pagination API is called

---
 packages/api/src/apollo.ts                    |   2 +
 packages/api/src/generated/graphql.ts         |  10 +-
 packages/api/src/generated/schema.graphql     |   5 +-
 packages/api/src/jobs/score_library_item.ts   |   2 +
 .../api/src/jobs/update_just_read_feed.ts     | 150 ++++++++++--------
 .../api/src/resolvers/function_resolvers.ts   |  11 ++
 .../api/src/resolvers/just_read_feed/index.ts |  23 ++-
 packages/api/src/resolvers/types.ts           |   2 +
 packages/api/src/schema.ts                    |   5 +-
 packages/api/src/services/just_read_feed.ts   |  59 +++++++
 packages/api/src/services/score.ts            |  50 +++---
 packages/api/src/services/subscriptions.ts    |  12 +-
 12 files changed, 227 insertions(+), 104 deletions(-)

diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts
index 11008c18b..76d7681be 100644
--- a/packages/api/src/apollo.ts
+++ b/packages/api/src/apollo.ts
@@ -32,6 +32,7 @@ import { ClaimsToSet, RequestContext, ResolverContext } from './resolvers/types'
 import ScalarResolvers from './scalars'
 import typeDefs from './schema'
 import { batchGetHighlightsFromLibraryItemIds } from './services/highlights'
+import { batchGetJustReadFeedItems } from './services/just_read_feed'
 import { batchGetLabelsFromLibraryItemIds } from './services/labels'
 import { batchGetRecommendationsFromLibraryItemIds } from './services/recommendation'
 import {
@@ -112,6 +113,7 @@ const contextFunc: ContextFunction<ExpressContext, ResolverContext> = async ({
         batchGetRecommendationsFromLibraryItemIds
       ),
       uploadFiles: new DataLoader(batchGetUploadFilesByIds),
+      justReadFeedItems: new DataLoader(batchGetJustReadFeedItems),
     },
   }
 
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index b3e7b91c3..99b993511 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1391,11 +1391,10 @@ export type JustReadFeedItem = {
   canShare?: Maybe<Scalars['Boolean']>;
   date: Scalars['Date'];
   dir?: Maybe<Scalars['String']>;
-  highlights?: Maybe<Scalars['String']>;
   id: Scalars['ID'];
-  likedCount?: Maybe<Scalars['Int']>;
+  likeCount?: Maybe<Scalars['Int']>;
   previewContent?: Maybe<Scalars['String']>;
-  savedCount?: Maybe<Scalars['Int']>;
+  saveCount?: Maybe<Scalars['Int']>;
   seen_at?: Maybe<Scalars['Date']>;
   subscription: JustReadFeedSubscription;
   thumbnail?: Maybe<Scalars['String']>;
@@ -6103,11 +6102,10 @@ export type JustReadFeedItemResolvers<ContextType = ResolverContext, ParentType
   canShare?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
   date?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
   dir?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  highlights?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
-  likedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  likeCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  savedCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  saveCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
   subscription?: Resolver<ResolversTypes['JustReadFeedSubscription'], ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 6a31ae89e..c79c05d23 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1253,11 +1253,10 @@ type JustReadFeedItem {
   canShare: Boolean
   date: Date!
   dir: String
-  highlights: String
   id: ID!
-  likedCount: Int
+  likeCount: Int
   previewContent: String
-  savedCount: Int
+  saveCount: Int
   seen_at: Date
   subscription: JustReadFeedSubscription!
   thumbnail: String
diff --git a/packages/api/src/jobs/score_library_item.ts b/packages/api/src/jobs/score_library_item.ts
index 5708316c6..3e001bf45 100644
--- a/packages/api/src/jobs/score_library_item.ts
+++ b/packages/api/src/jobs/score_library_item.ts
@@ -53,6 +53,7 @@ export const scoreLibraryItem = async (
       author: libraryItem.author,
       language: lanaugeToCode(libraryItem.itemLanguage || 'English'),
       word_count: libraryItem.wordCount,
+      published_at: libraryItem.publishedAt,
     } as Feature,
   }
 
@@ -77,4 +78,5 @@ export const scoreLibraryItem = async (
     undefined,
     true
   )
+  logger.info('Library item scored', data)
 }
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index e2d5c5e38..267dfcfb3 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -1,11 +1,12 @@
 import { LibraryItem } from '../entity/library_item'
 import { PublicItem } from '../entity/public_item'
+import { Subscription } from '../entity/subscription'
 import { User } from '../entity/user'
-import { JustReadFeedSection } from '../generated/graphql'
 import { redisDataSource } from '../redis_data_source'
 import { findUnseenPublicItems } from '../services/just_read_feed'
 import { searchLibraryItems } from '../services/library_item'
 import { Feature, getScores, ScoreApiResponse } from '../services/score'
+import { findSubscriptionsByNames } from '../services/subscriptions'
 import { findActiveUser } from '../services/user'
 import { lanaugeToCode } from '../utils/helpers'
 import { logger } from '../utils/logger'
@@ -16,10 +17,11 @@ export interface UpdateJustReadFeedJobData {
   userId: string
 }
 
-interface FeedItem {
+interface Candidate {
   id: string
   title: string
   url: string
+  type: string
   thumbnail?: string
   previewContent?: string
   languageCode: string
@@ -27,27 +29,36 @@ interface FeedItem {
   dir: string
   date: Date
   topic?: string
-  wordCount?: number
+  wordCount: number
   siteIcon?: string
   siteName?: string
-  saveCount?: number
-  likeCount?: number
-  broadcastCount?: number
   folder?: string
-  subscriptionType: string
   score?: number
   publishedAt?: Date
-  subscription: {
-    id: string
+  subscription?: {
     name: string
-    icon?: string
+    type: string
   }
 }
 
-const libraryItemToCandidate = (user: User, item: LibraryItem): FeedItem => ({
+interface Item {
+  id: string
+  type: string
+}
+
+interface Section {
+  items: Array<Item>
+  layout: string
+}
+
+const libraryItemToCandidate = (
+  item: LibraryItem,
+  subscriptions: Array<Subscription>
+): Candidate => ({
   id: item.id,
   title: item.title,
   url: item.originalUrl,
+  type: 'library_item',
   thumbnail: item.thumbnail || undefined,
   previewContent: item.description || undefined,
   languageCode: lanaugeToCode(item.itemLanguage || 'English'),
@@ -55,24 +66,24 @@ const libraryItemToCandidate = (user: User, item: LibraryItem): FeedItem => ({
   dir: item.directionality || 'ltr',
   date: item.createdAt,
   topic: item.topic,
-  wordCount: item.wordCount || undefined,
+  wordCount: item.wordCount!,
   siteName: item.siteName || undefined,
   siteIcon: item.siteIcon || undefined,
   folder: item.folder,
-  subscriptionType: 'library',
   score: item.score,
   publishedAt: item.publishedAt || undefined,
-  subscription: {
-    id: user.id,
-    name: user.name,
-    icon: user.profile.pictureUrl || undefined,
-  },
+  subscription: subscriptions.find(
+    (subscription) =>
+      subscription.name === item.subscription ||
+      subscription.url === item.subscription
+  ),
 })
 
-const publicItemToCandidate = (item: PublicItem): FeedItem => ({
+const publicItemToCandidate = (item: PublicItem): Candidate => ({
   id: item.id,
   title: item.title,
   url: item.url,
+  type: 'public_item',
   thumbnail: item.thumbnail,
   previewContent: item.previewContent,
   languageCode: item.languageCode || 'en',
@@ -80,38 +91,42 @@ const publicItemToCandidate = (item: PublicItem): FeedItem => ({
   dir: item.dir || 'ltr',
   date: item.createdAt,
   topic: item.topic,
-  wordCount: item.wordCount,
-  siteName: item.siteName,
-  saveCount: item.stats.saveCount,
-  likeCount: item.stats.likeCount,
-  broadcastCount: item.stats.broadcastCount,
+  wordCount: item.wordCount!,
   siteIcon: item.siteIcon,
-  subscriptionType: item.type,
   publishedAt: item.publishedAt,
   subscription: {
-    id: item.source.id,
     name: item.source.name,
-    icon: item.source.icon,
+    type: item.source.type,
   },
 })
 
-const selectCandidates = async (user: User): Promise<Array<FeedItem>> => {
+const selectCandidates = async (user: User): Promise<Array<Candidate>> => {
   const userId = user.id
   // get last 100 library items saved and not seen by user
   const libraryItems = await searchLibraryItems(
     {
       size: 100,
       includeContent: false,
-      query: `-is:seen`,
+      query: `-is:seen wordsCount:>0`,
     },
     userId
   )
 
   logger.info(`Found ${libraryItems.length} library items`)
 
+  // get subscriptions for the library items
+  const subscriptionNames = libraryItems
+    .filter((item) => !!item.subscription)
+    .map((item) => item.subscription as string)
+
+  const subscriptions = await findSubscriptionsByNames(
+    userId,
+    subscriptionNames
+  )
+
   // map library items to candidates and limit to 70
-  const privateCandidates: Array<FeedItem> = libraryItems
-    .map((libraryItem) => libraryItemToCandidate(user, libraryItem))
+  const privateCandidates: Array<Candidate> = libraryItems
+    .map((item) => libraryItemToCandidate(item, subscriptions))
     .slice(0, 70)
   const privateCandidatesSize = privateCandidates.length
 
@@ -125,7 +140,7 @@ const selectCandidates = async (user: User): Promise<Array<FeedItem>> => {
   logger.info(`Found ${publicItems.length} public items`)
 
   // map public items to candidates and limit to the remaining vacancies
-  const publicCandidates: Array<FeedItem> = publicItems
+  const publicCandidates: Array<Candidate> = publicItems
     .map(publicItemToCandidate)
     .slice(0, 100 - privateCandidatesSize)
 
@@ -138,8 +153,8 @@ const selectCandidates = async (user: User): Promise<Array<FeedItem>> => {
 
 const rankCandidates = async (
   userId: string,
-  candidates: Array<FeedItem>
-): Promise<Array<FeedItem>> => {
+  candidates: Array<Candidate>
+): Promise<Array<Candidate>> => {
   if (candidates.length <= 10) {
     // no need to rank if there are less than 10 candidates
     return candidates
@@ -155,13 +170,13 @@ const rankCandidates = async (
       acc[item.id] = {
         title: item.title,
         has_thumbnail: !!item.thumbnail,
-        has_site_icon: !!item.subscription.icon,
+        has_site_icon: !!item.siteIcon,
         saved_at: item.date,
         site: item.siteName,
         language: item.languageCode,
         directionality: item.dir,
-        folder: item.subscription.name,
-        subscription_type: item.subscriptionType,
+        folder: item.folder,
+        subscription_type: item.subscription?.type,
         author: item.author,
         word_count: item.wordCount,
         published_at: item.publishedAt,
@@ -196,8 +211,8 @@ const MAX_FEED_ITEMS = 500
 export const getJustReadFeedSections = async (
   userId: string,
   limit: number,
-  minScore?: number
-): Promise<Array<{ member: JustReadFeedSection; score: number }>> => {
+  maxScore?: number
+): Promise<Array<{ member: Section; score: number }>> => {
   const redisClient = redisDataSource.redisClient
   if (!redisClient) {
     throw new Error('Redis client not available')
@@ -211,8 +226,8 @@ export const getJustReadFeedSections = async (
   // response is an array of [member1, score1, member2, score2, ...]
   const results = await redisClient.zrevrangebyscore(
     key,
-    '+inf',
-    minScore || '-inf',
+    maxScore ? maxScore - 1 : '+inf',
+    '-inf',
     'WITHSCORES',
     'LIMIT',
     0,
@@ -221,7 +236,7 @@ export const getJustReadFeedSections = async (
 
   const sections = []
   for (let i = 0; i < results.length; i += 2) {
-    const member = JSON.parse(results[i]) as JustReadFeedSection
+    const member = JSON.parse(results[i]) as Section
     const score = Number(results[i + 1])
     sections.push({ member, score })
   }
@@ -231,7 +246,7 @@ export const getJustReadFeedSections = async (
 
 const appendSectionsToFeed = async (
   userId: string,
-  sections: Array<JustReadFeedSection>
+  sections: Array<Section>
 ) => {
   const redisClient = redisDataSource.redisClient
   if (!redisClient) {
@@ -243,8 +258,8 @@ const appendSectionsToFeed = async (
   // store candidates in redis sorted set
   const pipeline = redisClient.pipeline()
 
-  const scoreMembers = sections.flatMap((section) => [
-    Date.now() + 86_400_000, // sections expire in 24 hours
+  const scoreMembers = sections.flatMap((section, index) => [
+    Date.now() + index + 86_400_000, // sections expire in 24 hours
     JSON.stringify(section),
   ])
   // add section to the sorted set
@@ -258,32 +273,33 @@ const appendSectionsToFeed = async (
   await pipeline.exec()
 }
 
-const mixFeedItems = (
-  rankedFeedItems: Array<FeedItem>
-): Array<JustReadFeedSection> => {
+const mixFeedItems = (rankedFeedItems: Array<Candidate>): Array<Section> => {
   // find the median word count
-  const wordCounts = rankedFeedItems.map((item) => item.wordCount || 0)
-  wordCounts.sort()
+  const wordCounts = rankedFeedItems.map((item) => item.wordCount)
+  wordCounts.sort((a, b) => a - b)
   const medianWordCount = wordCounts[Math.floor(wordCounts.length / 2)]
   // separate items into two groups based on word count
-  const shortItems: Array<FeedItem> = []
-  const longItems: Array<FeedItem> = []
+  const shortItems: Array<Candidate> = []
+  const longItems: Array<Candidate> = []
   for (const item of rankedFeedItems) {
-    if (item.wordCount && item.wordCount < medianWordCount) {
+    if (item.wordCount < medianWordCount) {
       shortItems.push(item)
     } else {
       longItems.push(item)
     }
   }
   // initialize empty batches
-  const batches = [[]]
+  const batches: Array<Array<Candidate>> = Array.from(
+    { length: Math.floor(rankedFeedItems.length / 10) },
+    () => []
+  )
 
-  const checkConstraints = (batch: Array<FeedItem>, item: FeedItem) => {
+  const checkConstraints = (batch: Array<Candidate>, item: Candidate) => {
     const titleCount = batch.filter((i) => i.title === item.title).length
     const authorCount = batch.filter((i) => i.author === item.author).length
     const siteCount = batch.filter((i) => i.siteName === item.siteName).length
     const subscriptionCount = batch.filter(
-      (i) => i.subscription.name === item.subscription.name
+      (i) => i.subscription?.name === item.subscription?.name
     ).length
 
     return (
@@ -295,8 +311,8 @@ const mixFeedItems = (
   }
 
   const distributeItems = (
-    items: Array<FeedItem>,
-    batches: Array<Array<FeedItem>>
+    items: Array<Candidate>,
+    batches: Array<Array<Candidate>>
   ) => {
     for (const item of items) {
       let added = false
@@ -310,7 +326,7 @@ const mixFeedItems = (
 
       if (!added) {
         for (const batch of batches) {
-          if (batch.length < 5) {
+          if (batch.length < 10) {
             batch.push(item)
             break
           }
@@ -328,15 +344,23 @@ const mixFeedItems = (
   for (const batch of batches) {
     // create a section for each long item
     for (let i = 0; i < 5; i++) {
-      const section: JustReadFeedSection = {
-        items: [batch[i]],
+      const section: Section = {
+        items: [
+          {
+            id: batch[i].id,
+            type: batch[i].type,
+          },
+        ],
         layout: 'long',
       }
       sections.push(section)
     }
     // create a section for short items
     sections.push({
-      items: batch.slice(5),
+      items: batch.slice(5).map((item) => ({
+        id: item.id,
+        type: item.type,
+      })),
       layout: 'quick links',
     })
   }
@@ -372,7 +396,7 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
   const rankedSections = mixFeedItems(rankedCandidates)
   logger.info(`Created ${rankedSections.length} sections`)
 
-  logger.info('Appending feed items to feed')
+  logger.info('Appending sections to feed')
   await appendSectionsToFeed(userId, rankedSections)
   logger.info('Feed updated for user', { userId })
 }
diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts
index 91f8b942f..13a750a9f 100644
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@@ -624,6 +624,17 @@ export const functionResolvers = {
       return newsletterEmail.folder || EXISTING_NEWSLETTER_FOLDER
     },
   },
+  JustReadFeedSection: {
+    async items(
+      section: { items: Array<{ id: string }> },
+      _: unknown,
+      ctx: WithDataSourcesContext
+    ) {
+      return ctx.dataLoaders.justReadFeedItems.loadMany(
+        section.items.map((i) => i.id)
+      )
+    },
+  },
   ...resultResolveTypeResolver('Login'),
   ...resultResolveTypeResolver('LogOut'),
   ...resultResolveTypeResolver('GoogleSignup'),
diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/just_read_feed/index.ts
index e6ff119a6..1a225dc87 100644
--- a/packages/api/src/resolvers/just_read_feed/index.ts
+++ b/packages/api/src/resolvers/just_read_feed/index.ts
@@ -1,29 +1,46 @@
 import {
   JustReadFeedError,
   JustReadFeedErrorCode,
+  JustReadFeedItem,
+  JustReadFeedSection,
   JustReadFeedSuccess,
   QueryJustReadFeedArgs,
 } from '../../generated/graphql'
 import { getJustReadFeedSections } from '../../jobs/update_just_read_feed'
 import { getJob } from '../../queue-processor'
+import { Merge } from '../../util'
 import {
   enqueueUpdateJustReadFeed,
   updateJustReadFeedJobId,
 } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 
+type PartialJustReadFeedItem = Merge<
+  Partial<JustReadFeedItem>,
+  { type: string }
+>
+type PartialJustReadFeedSection = Merge<
+  JustReadFeedSection,
+  { items: Array<PartialJustReadFeedItem> }
+>
+type PartialJustReadFeedSuccess = Merge<
+  JustReadFeedSuccess,
+  {
+    edges: Array<{ cursor: string; node: PartialJustReadFeedSection }>
+  }
+>
 // This resolver is used to fetch the just read feed for the user.
 // when the feed is empty, it enqueues a job to update the feed.
 // when client tries to fetch more then the feed has, it enqueues a job to update the feed.
 export const justReadFeedResolver = authorized<
-  JustReadFeedSuccess,
+  PartialJustReadFeedSuccess,
   JustReadFeedError,
   QueryJustReadFeedArgs
 >(async (_, { first, after }, { uid, log }) => {
   const limit = first || 10
-  const minScore = after ? Number(after) : 0
+  const cursor = after ? parseInt(after) : undefined
 
-  const sections = await getJustReadFeedSections(uid, limit, minScore)
+  const sections = await getJustReadFeedSections(uid, limit, cursor)
   log.info('Just read feed sections fetched')
 
   if (sections.length === 0) {
diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts
index f880a0145..dacc50476 100644
--- a/packages/api/src/resolvers/types.ts
+++ b/packages/api/src/resolvers/types.ts
@@ -10,6 +10,7 @@ import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
 import { Recommendation } from '../entity/recommendation'
 import { UploadFile } from '../entity/upload_file'
+import { JustReadFeedItem } from '../generated/graphql'
 import { PubsubClient } from '../pubsub'
 
 export interface Claims {
@@ -51,6 +52,7 @@ export interface RequestContext {
     highlights: DataLoader<string, Highlight[]>
     recommendations: DataLoader<string, Recommendation[]>
     uploadFiles: DataLoader<string, UploadFile | undefined>
+    justReadFeedItems: DataLoader<string, JustReadFeedItem>
   }
 }
 
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 3a7c0bd4c..c0b9345cb 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3114,9 +3114,8 @@ const schema = gql`
     url: String!
     thumbnail: String
     previewContent: String
-    highlights: String
-    savedCount: Int
-    likedCount: Int
+    saveCount: Int
+    likeCount: Int
     broadcastCount: Int
     date: Date!
     author: String
diff --git a/packages/api/src/services/just_read_feed.ts b/packages/api/src/services/just_read_feed.ts
index 895192062..a38da1650 100644
--- a/packages/api/src/services/just_read_feed.ts
+++ b/packages/api/src/services/just_read_feed.ts
@@ -1,5 +1,64 @@
 import { PublicItem } from '../entity/public_item'
+import { JustReadFeedItem } from '../generated/graphql'
 import { authTrx } from '../repository'
+import { findLibraryItemsByIds } from './library_item'
+
+export const batchGetJustReadFeedItems = async (
+  ids: readonly string[]
+): Promise<Array<JustReadFeedItem>> => {
+  const libraryItems = await findLibraryItemsByIds(ids as string[], '')
+
+  const publicItems = await authTrx(async (tx) =>
+    tx
+      .getRepository(PublicItem)
+      .createQueryBuilder('public_item')
+      .innerJoin(
+        'public_item_stats',
+        'stats',
+        'stats.public_item_id = public_item.id'
+      )
+      .innerJoin(
+        'public_item_source',
+        'source',
+        'source.id = public_item.source_id'
+      )
+      .where('public_item.id IN (:...ids)', { ids })
+      .getMany()
+  )
+
+  return ids
+    .map((id) => {
+      const libraryItem = libraryItems.find((li) => li.id === id)
+      if (libraryItem) {
+        return {
+          ...libraryItem,
+          date: libraryItem.savedAt,
+          url: libraryItem.originalUrl,
+          canArchive: !libraryItem.archivedAt,
+          canDelete: !libraryItem.deletedAt,
+          canSave: false,
+          dir: libraryItem.directionality,
+        }
+      } else {
+        const publicItem = publicItems.find((pi) => pi.id === id)
+        return publicItem
+          ? {
+              ...publicItem,
+              date: publicItem.createdAt,
+              url: publicItem.url,
+              canArchive: false,
+              canDelete: false,
+              canSave: true,
+              broadcastCount: publicItem.stats.broadcastCount,
+              likeCount: publicItem.stats.likeCount,
+              saveCount: publicItem.stats.saveCount,
+              subscription: publicItem.source,
+            }
+          : undefined
+      }
+    })
+    .filter((item) => item !== undefined) as JustReadFeedItem[]
+}
 
 export const findUnseenPublicItems = async (
   userId: string,
diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
index ed1820efc..e2dc8d1ca 100644
--- a/packages/api/src/services/score.ts
+++ b/packages/api/src/services/score.ts
@@ -8,8 +8,8 @@ export interface Feature {
   author?: string
   directionality: string
   word_count?: number
-  subscription_type: string
-  folder: string
+  subscription_type?: string
+  folder?: string
   published_at?: Date
 }
 
@@ -23,33 +23,33 @@ export type ScoreApiResponse = Record<string, number> // item_id -> score
 export const getScores = async (
   data: ScoreApiRequestBody
 ): Promise<ScoreApiResponse> => {
-  const API_URL = 'http://127.0.0.1:5000/predictions'
-  const token = process.env.SCORE_API_TOKEN
+  // const API_URL = 'http://127.0.0.1:5000/predictions'
+  // const token = process.env.SCORE_API_TOKEN
 
-  if (!token) {
-    throw new Error('No score API token found')
-  }
+  // if (!token) {
+  //   throw new Error('No score API token found')
+  // }
 
-  const response = await fetch(API_URL, {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      Authorization: `Bearer ${token}`,
-    },
-    body: JSON.stringify(data),
-  })
+  // const response = await fetch(API_URL, {
+  //   method: 'POST',
+  //   headers: {
+  //     'Content-Type': 'application/json',
+  //     Authorization: `Bearer ${token}`,
+  //   },
+  //   body: JSON.stringify(data),
+  // })
 
-  if (!response.ok) {
-    throw new Error(`Failed to score candidates: ${response.statusText}`)
-  }
+  // if (!response.ok) {
+  //   throw new Error(`Failed to score candidates: ${response.statusText}`)
+  // }
 
-  const scores = (await response.json()) as ScoreApiResponse
-  return scores
+  // const scores = (await response.json()) as ScoreApiResponse
+  // return scores
 
   // fake random scores
-  // const scores: ScoreApiResponse = {}
-  // for (const itemId of Object.keys(data.item_features)) {
-  //   scores[itemId] = Math.random()
-  // }
-  // return Promise.resolve(scores)
+  const scores: ScoreApiResponse = {}
+  for (const itemId of Object.keys(data.item_features)) {
+    scores[itemId] = Math.random()
+  }
+  return Promise.resolve(scores)
 }
diff --git a/packages/api/src/services/subscriptions.ts b/packages/api/src/services/subscriptions.ts
index 22be8c2f4..72b0fc9d2 100644
--- a/packages/api/src/services/subscriptions.ts
+++ b/packages/api/src/services/subscriptions.ts
@@ -1,5 +1,5 @@
 import axios from 'axios'
-import { DeepPartial, DeleteResult } from 'typeorm'
+import { DeepPartial, DeleteResult, In } from 'typeorm'
 import { appDataSource } from '../data_source'
 import { NewsletterEmail } from '../entity/newsletter_email'
 import { Subscription } from '../entity/subscription'
@@ -214,3 +214,13 @@ export const createRssSubscriptions = async (
 ) => {
   return getRepository(Subscription).save(subscriptions)
 }
+
+export const findSubscriptionsByNames = async (
+  userId: string,
+  names: string[]
+): Promise<Subscription[]> => {
+  return getRepository(Subscription).findBy([
+    { user: { id: userId }, name: In(names) },
+    { user: { id: userId }, url: In(names) },
+  ])
+}

From b24d8ca41328fb9e7b3dec3bb1786c4b8b4c2963 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 17:55:19 +0800
Subject: [PATCH 24/33] show long read items before quick links

---
 .../api/src/jobs/update_just_read_feed.ts     | 31 +++++------
 packages/api/src/services/score.ts            | 54 +++++++++----------
 2 files changed, 40 insertions(+), 45 deletions(-)

diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_just_read_feed.ts
index 267dfcfb3..5f2cecc59 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_just_read_feed.ts
@@ -221,7 +221,7 @@ export const getJustReadFeedSections = async (
   const key = redisKey(userId)
 
   // get feed items from redis sorted set in descending order
-  // with score greater than minScore
+  // with score smalled than maxScore
   // limit to the first `limit` items
   // response is an array of [member1, score1, member2, score2, ...]
   const results = await redisClient.zrevrangebyscore(
@@ -335,34 +335,29 @@ const mixFeedItems = (rankedFeedItems: Array<Candidate>): Array<Section> => {
     }
   }
 
-  // distribute longer items first and then shorter items
-  distributeItems(longItems, batches)
+  // distribute quick link items first
   distributeItems(shortItems, batches)
+  distributeItems(longItems, batches)
 
   // convert batches to sections
   const sections = []
   for (const batch of batches) {
-    // create a section for each long item
-    for (let i = 0; i < 5; i++) {
-      const section: Section = {
-        items: [
-          {
-            id: batch[i].id,
-            type: batch[i].type,
-          },
-        ],
-        layout: 'long',
-      }
-      sections.push(section)
-    }
-    // create a section for short items
+    // create a section for all quick links
     sections.push({
-      items: batch.slice(5).map((item) => ({
+      items: batch.slice(0, 5).map((item) => ({
         id: item.id,
         type: item.type,
       })),
       layout: 'quick links',
     })
+
+    // create a section for each long item
+    sections.push(
+      ...batch.slice(5).map((item) => ({
+        items: [{ id: item.id, type: item.type }],
+        layout: 'long',
+      }))
+    )
   }
 
   return sections
diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
index e2dc8d1ca..cd9cf9dc1 100644
--- a/packages/api/src/services/score.ts
+++ b/packages/api/src/services/score.ts
@@ -23,33 +23,33 @@ export type ScoreApiResponse = Record<string, number> // item_id -> score
 export const getScores = async (
   data: ScoreApiRequestBody
 ): Promise<ScoreApiResponse> => {
-  // const API_URL = 'http://127.0.0.1:5000/predictions'
-  // const token = process.env.SCORE_API_TOKEN
+  const API_URL = 'http://127.0.0.1:5000/predictions'
+  const token = process.env.SCORE_API_TOKEN
 
-  // if (!token) {
-  //   throw new Error('No score API token found')
-  // }
-
-  // const response = await fetch(API_URL, {
-  //   method: 'POST',
-  //   headers: {
-  //     'Content-Type': 'application/json',
-  //     Authorization: `Bearer ${token}`,
-  //   },
-  //   body: JSON.stringify(data),
-  // })
-
-  // if (!response.ok) {
-  //   throw new Error(`Failed to score candidates: ${response.statusText}`)
-  // }
-
-  // const scores = (await response.json()) as ScoreApiResponse
-  // return scores
-
-  // fake random scores
-  const scores: ScoreApiResponse = {}
-  for (const itemId of Object.keys(data.item_features)) {
-    scores[itemId] = Math.random()
+  if (!token) {
+    throw new Error('No score API token found')
   }
-  return Promise.resolve(scores)
+
+  const response = await fetch(API_URL, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${token}`,
+    },
+    body: JSON.stringify(data),
+  })
+
+  if (!response.ok) {
+    throw new Error(`Failed to score candidates: ${response.statusText}`)
+  }
+
+  const scores = (await response.json()) as ScoreApiResponse
+  return scores
+
+  // // fake random scores
+  // const scores: ScoreApiResponse = {}
+  // for (const itemId of Object.keys(data.item_features)) {
+  //   scores[itemId] = Math.random()
+  // }
+  // return Promise.resolve(scores)
 }

From 5b9e34739df61fca18d98e99311f2bceded0f4c3 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 17:59:04 +0800
Subject: [PATCH 25/33] skip auth token for score api for now

---
 packages/api/src/services/score.ts | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
index cd9cf9dc1..4a4bbf6d5 100644
--- a/packages/api/src/services/score.ts
+++ b/packages/api/src/services/score.ts
@@ -24,17 +24,17 @@ export const getScores = async (
   data: ScoreApiRequestBody
 ): Promise<ScoreApiResponse> => {
   const API_URL = 'http://127.0.0.1:5000/predictions'
-  const token = process.env.SCORE_API_TOKEN
+  // const token = process.env.SCORE_API_TOKEN
 
-  if (!token) {
-    throw new Error('No score API token found')
-  }
+  // if (!token) {
+  //   throw new Error('No score API token found')
+  // }
 
   const response = await fetch(API_URL, {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json',
-      Authorization: `Bearer ${token}`,
+      // Authorization: `Bearer ${token}`,
     },
     body: JSON.stringify(data),
   })

From efb7160373a448e59c9806be13f5131c765a38df Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 21:11:08 +0800
Subject: [PATCH 26/33] rename just read feed to home

---
 packages/api/src/apollo.ts                    |   4 +-
 packages/api/src/generated/graphql.ts         | 306 +++++++++---------
 packages/api/src/generated/schema.graphql     | 118 +++----
 ...pdate_just_read_feed.ts => update_home.ts} |  34 +-
 packages/api/src/queue-processor.ts           |   9 +-
 .../api/src/resolvers/function_resolvers.ts   |  12 +-
 .../{just_read_feed => home}/index.ts         |  53 ++-
 packages/api/src/resolvers/types.ts           |   4 +-
 packages/api/src/schema.ts                    |  28 +-
 .../services/{just_read_feed.ts => home.ts}   |  10 +-
 packages/api/src/services/library_item.ts     |   2 +-
 packages/api/src/utils/createTask.ts          |  21 +-
 12 files changed, 291 insertions(+), 310 deletions(-)
 rename packages/api/src/jobs/{update_just_read_feed.ts => update_home.ts} (91%)
 rename packages/api/src/resolvers/{just_read_feed => home}/index.ts (52%)
 rename packages/api/src/services/{just_read_feed.ts => home.ts} (92%)

diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts
index 76d7681be..d4885fa7e 100644
--- a/packages/api/src/apollo.ts
+++ b/packages/api/src/apollo.ts
@@ -32,7 +32,7 @@ import { ClaimsToSet, RequestContext, ResolverContext } from './resolvers/types'
 import ScalarResolvers from './scalars'
 import typeDefs from './schema'
 import { batchGetHighlightsFromLibraryItemIds } from './services/highlights'
-import { batchGetJustReadFeedItems } from './services/just_read_feed'
+import { batchGetHomeItems } from './services/home'
 import { batchGetLabelsFromLibraryItemIds } from './services/labels'
 import { batchGetRecommendationsFromLibraryItemIds } from './services/recommendation'
 import {
@@ -113,7 +113,7 @@ const contextFunc: ContextFunction<ExpressContext, ResolverContext> = async ({
         batchGetRecommendationsFromLibraryItemIds
       ),
       uploadFiles: new DataLoader(batchGetUploadFilesByIds),
-      justReadFeedItems: new DataLoader(batchGetJustReadFeedItems),
+      homeItems: new DataLoader(batchGetHomeItems),
     },
   }
 
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 99b993511..a3f839346 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1270,6 +1270,70 @@ export enum HighlightType {
   Redaction = 'REDACTION'
 }
 
+export type HomeEdge = {
+  __typename?: 'HomeEdge';
+  cursor: Scalars['String'];
+  node: HomeSection;
+};
+
+export type HomeError = {
+  __typename?: 'HomeError';
+  errorCodes: Array<HomeErrorCode>;
+};
+
+export enum HomeErrorCode {
+  BadRequest = 'BAD_REQUEST',
+  Pending = 'PENDING',
+  Unauthorized = 'UNAUTHORIZED'
+}
+
+export type HomeItem = {
+  __typename?: 'HomeItem';
+  author?: Maybe<Scalars['String']>;
+  broadcastCount?: Maybe<Scalars['Int']>;
+  canArchive?: Maybe<Scalars['Boolean']>;
+  canComment?: Maybe<Scalars['Boolean']>;
+  canDelete?: Maybe<Scalars['Boolean']>;
+  canSave?: Maybe<Scalars['Boolean']>;
+  canShare?: Maybe<Scalars['Boolean']>;
+  date: Scalars['Date'];
+  dir?: Maybe<Scalars['String']>;
+  id: Scalars['ID'];
+  likeCount?: Maybe<Scalars['Int']>;
+  previewContent?: Maybe<Scalars['String']>;
+  saveCount?: Maybe<Scalars['Int']>;
+  seen_at?: Maybe<Scalars['Date']>;
+  subscription: HomeSubscription;
+  thumbnail?: Maybe<Scalars['String']>;
+  title: Scalars['String'];
+  url: Scalars['String'];
+  wordCount?: Maybe<Scalars['Int']>;
+};
+
+export type HomeResult = HomeError | HomeSuccess;
+
+export type HomeSection = {
+  __typename?: 'HomeSection';
+  items: Array<HomeItem>;
+  layout?: Maybe<Scalars['String']>;
+  thumbnail?: Maybe<Scalars['String']>;
+  title?: Maybe<Scalars['String']>;
+};
+
+export type HomeSubscription = {
+  __typename?: 'HomeSubscription';
+  icon?: Maybe<Scalars['String']>;
+  id: Scalars['ID'];
+  name: Scalars['String'];
+  url?: Maybe<Scalars['String']>;
+};
+
+export type HomeSuccess = {
+  __typename?: 'HomeSuccess';
+  edges: Array<HomeEdge>;
+  pageInfo: PageInfo;
+};
+
 export type ImportFromIntegrationError = {
   __typename?: 'ImportFromIntegrationError';
   errorCodes: Array<ImportFromIntegrationErrorCode>;
@@ -1363,70 +1427,6 @@ export type JoinGroupSuccess = {
   group: RecommendationGroup;
 };
 
-export type JustReadFeedEdge = {
-  __typename?: 'JustReadFeedEdge';
-  cursor: Scalars['String'];
-  node: JustReadFeedSection;
-};
-
-export type JustReadFeedError = {
-  __typename?: 'JustReadFeedError';
-  errorCodes: Array<JustReadFeedErrorCode>;
-};
-
-export enum JustReadFeedErrorCode {
-  BadRequest = 'BAD_REQUEST',
-  Pending = 'PENDING',
-  Unauthorized = 'UNAUTHORIZED'
-}
-
-export type JustReadFeedItem = {
-  __typename?: 'JustReadFeedItem';
-  author?: Maybe<Scalars['String']>;
-  broadcastCount?: Maybe<Scalars['Int']>;
-  canArchive?: Maybe<Scalars['Boolean']>;
-  canComment?: Maybe<Scalars['Boolean']>;
-  canDelete?: Maybe<Scalars['Boolean']>;
-  canSave?: Maybe<Scalars['Boolean']>;
-  canShare?: Maybe<Scalars['Boolean']>;
-  date: Scalars['Date'];
-  dir?: Maybe<Scalars['String']>;
-  id: Scalars['ID'];
-  likeCount?: Maybe<Scalars['Int']>;
-  previewContent?: Maybe<Scalars['String']>;
-  saveCount?: Maybe<Scalars['Int']>;
-  seen_at?: Maybe<Scalars['Date']>;
-  subscription: JustReadFeedSubscription;
-  thumbnail?: Maybe<Scalars['String']>;
-  title: Scalars['String'];
-  url: Scalars['String'];
-  wordCount?: Maybe<Scalars['Int']>;
-};
-
-export type JustReadFeedResult = JustReadFeedError | JustReadFeedSuccess;
-
-export type JustReadFeedSection = {
-  __typename?: 'JustReadFeedSection';
-  items: Array<JustReadFeedItem>;
-  layout?: Maybe<Scalars['String']>;
-  thumbnail?: Maybe<Scalars['String']>;
-  title?: Maybe<Scalars['String']>;
-};
-
-export type JustReadFeedSubscription = {
-  __typename?: 'JustReadFeedSubscription';
-  icon?: Maybe<Scalars['String']>;
-  id: Scalars['ID'];
-  name: Scalars['String'];
-  url?: Maybe<Scalars['String']>;
-};
-
-export type JustReadFeedSuccess = {
-  __typename?: 'JustReadFeedSuccess';
-  edges: Array<JustReadFeedEdge>;
-  pageInfo: PageInfo;
-};
-
 export type Label = {
   __typename?: 'Label';
   color: Scalars['String'];
@@ -2223,9 +2223,9 @@ export type Query = {
   getUserPersonalization: GetUserPersonalizationResult;
   groups: GroupsResult;
   hello?: Maybe<Scalars['String']>;
+  home: HomeResult;
   integration: IntegrationResult;
   integrations: IntegrationsResult;
-  justReadFeed: JustReadFeedResult;
   labels: LabelsResult;
   me?: Maybe<User>;
   newsletterEmails: NewsletterEmailsResult;
@@ -2272,14 +2272,14 @@ export type QueryGetDiscoverFeedArticlesArgs = {
 };
 
 
-export type QueryIntegrationArgs = {
-  name: Scalars['String'];
+export type QueryHomeArgs = {
+  after?: InputMaybe<Scalars['String']>;
+  first?: InputMaybe<Scalars['Int']>;
 };
 
 
-export type QueryJustReadFeedArgs = {
-  after?: InputMaybe<Scalars['String']>;
-  first?: InputMaybe<Scalars['Int']>;
+export type QueryIntegrationArgs = {
+  name: Scalars['String'];
 };
 
 
@@ -4255,6 +4255,14 @@ export type ResolversTypes = {
   HighlightReply: ResolverTypeWrapper<HighlightReply>;
   HighlightStats: ResolverTypeWrapper<HighlightStats>;
   HighlightType: HighlightType;
+  HomeEdge: ResolverTypeWrapper<HomeEdge>;
+  HomeError: ResolverTypeWrapper<HomeError>;
+  HomeErrorCode: HomeErrorCode;
+  HomeItem: ResolverTypeWrapper<HomeItem>;
+  HomeResult: ResolversTypes['HomeError'] | ResolversTypes['HomeSuccess'];
+  HomeSection: ResolverTypeWrapper<HomeSection>;
+  HomeSubscription: ResolverTypeWrapper<HomeSubscription>;
+  HomeSuccess: ResolverTypeWrapper<HomeSuccess>;
   ID: ResolverTypeWrapper<Scalars['ID']>;
   ImportFromIntegrationError: ResolverTypeWrapper<ImportFromIntegrationError>;
   ImportFromIntegrationErrorCode: ImportFromIntegrationErrorCode;
@@ -4277,14 +4285,6 @@ export type ResolversTypes = {
   JoinGroupErrorCode: JoinGroupErrorCode;
   JoinGroupResult: ResolversTypes['JoinGroupError'] | ResolversTypes['JoinGroupSuccess'];
   JoinGroupSuccess: ResolverTypeWrapper<JoinGroupSuccess>;
-  JustReadFeedEdge: ResolverTypeWrapper<JustReadFeedEdge>;
-  JustReadFeedError: ResolverTypeWrapper<JustReadFeedError>;
-  JustReadFeedErrorCode: JustReadFeedErrorCode;
-  JustReadFeedItem: ResolverTypeWrapper<JustReadFeedItem>;
-  JustReadFeedResult: ResolversTypes['JustReadFeedError'] | ResolversTypes['JustReadFeedSuccess'];
-  JustReadFeedSection: ResolverTypeWrapper<JustReadFeedSection>;
-  JustReadFeedSubscription: ResolverTypeWrapper<JustReadFeedSubscription>;
-  JustReadFeedSuccess: ResolverTypeWrapper<JustReadFeedSuccess>;
   Label: ResolverTypeWrapper<Label>;
   LabelsError: ResolverTypeWrapper<LabelsError>;
   LabelsErrorCode: LabelsErrorCode;
@@ -4812,6 +4812,13 @@ export type ResolversParentTypes = {
   Highlight: Highlight;
   HighlightReply: HighlightReply;
   HighlightStats: HighlightStats;
+  HomeEdge: HomeEdge;
+  HomeError: HomeError;
+  HomeItem: HomeItem;
+  HomeResult: ResolversParentTypes['HomeError'] | ResolversParentTypes['HomeSuccess'];
+  HomeSection: HomeSection;
+  HomeSubscription: HomeSubscription;
+  HomeSuccess: HomeSuccess;
   ID: Scalars['ID'];
   ImportFromIntegrationError: ImportFromIntegrationError;
   ImportFromIntegrationResult: ResolversParentTypes['ImportFromIntegrationError'] | ResolversParentTypes['ImportFromIntegrationSuccess'];
@@ -4828,13 +4835,6 @@ export type ResolversParentTypes = {
   JoinGroupError: JoinGroupError;
   JoinGroupResult: ResolversParentTypes['JoinGroupError'] | ResolversParentTypes['JoinGroupSuccess'];
   JoinGroupSuccess: JoinGroupSuccess;
-  JustReadFeedEdge: JustReadFeedEdge;
-  JustReadFeedError: JustReadFeedError;
-  JustReadFeedItem: JustReadFeedItem;
-  JustReadFeedResult: ResolversParentTypes['JustReadFeedError'] | ResolversParentTypes['JustReadFeedSuccess'];
-  JustReadFeedSection: JustReadFeedSection;
-  JustReadFeedSubscription: JustReadFeedSubscription;
-  JustReadFeedSuccess: JustReadFeedSuccess;
   Label: Label;
   LabelsError: LabelsError;
   LabelsResult: ResolversParentTypes['LabelsError'] | ResolversParentTypes['LabelsSuccess'];
@@ -6008,6 +6008,66 @@ export type HighlightStatsResolvers<ContextType = ResolverContext, ParentType ex
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type HomeEdgeResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeEdge'] = ResolversParentTypes['HomeEdge']> = {
+  cursor?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  node?: Resolver<ResolversTypes['HomeSection'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type HomeErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeError'] = ResolversParentTypes['HomeError']> = {
+  errorCodes?: Resolver<Array<ResolversTypes['HomeErrorCode']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type HomeItemResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeItem'] = ResolversParentTypes['HomeItem']> = {
+  author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  broadcastCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  canArchive?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  canComment?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  canDelete?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  canSave?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  canShare?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
+  date?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
+  dir?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  likeCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  saveCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
+  subscription?: Resolver<ResolversTypes['HomeSubscription'], ParentType, ContextType>;
+  thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  wordCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type HomeResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeResult'] = ResolversParentTypes['HomeResult']> = {
+  __resolveType: TypeResolveFn<'HomeError' | 'HomeSuccess', ParentType, ContextType>;
+};
+
+export type HomeSectionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeSection'] = ResolversParentTypes['HomeSection']> = {
+  items?: Resolver<Array<ResolversTypes['HomeItem']>, ParentType, ContextType>;
+  layout?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  title?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type HomeSubscriptionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeSubscription'] = ResolversParentTypes['HomeSubscription']> = {
+  icon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
+export type HomeSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeSuccess'] = ResolversParentTypes['HomeSuccess']> = {
+  edges?: Resolver<Array<ResolversTypes['HomeEdge']>, ParentType, ContextType>;
+  pageInfo?: Resolver<ResolversTypes['PageInfo'], ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type ImportFromIntegrationErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['ImportFromIntegrationError'] = ResolversParentTypes['ImportFromIntegrationError']> = {
   errorCodes?: Resolver<Array<ResolversTypes['ImportFromIntegrationErrorCode']>, ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
@@ -6081,66 +6141,6 @@ export type JoinGroupSuccessResolvers<ContextType = ResolverContext, ParentType
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
-export type JustReadFeedEdgeResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedEdge'] = ResolversParentTypes['JustReadFeedEdge']> = {
-  cursor?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  node?: Resolver<ResolversTypes['JustReadFeedSection'], ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
-export type JustReadFeedErrorResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedError'] = ResolversParentTypes['JustReadFeedError']> = {
-  errorCodes?: Resolver<Array<ResolversTypes['JustReadFeedErrorCode']>, ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
-export type JustReadFeedItemResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedItem'] = ResolversParentTypes['JustReadFeedItem']> = {
-  author?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  broadcastCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
-  canArchive?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
-  canComment?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
-  canDelete?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
-  canSave?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
-  canShare?: Resolver<Maybe<ResolversTypes['Boolean']>, ParentType, ContextType>;
-  date?: Resolver<ResolversTypes['Date'], ParentType, ContextType>;
-  dir?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
-  likeCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
-  previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  saveCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
-  seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
-  subscription?: Resolver<ResolversTypes['JustReadFeedSubscription'], ParentType, ContextType>;
-  thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  wordCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
-export type JustReadFeedResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedResult'] = ResolversParentTypes['JustReadFeedResult']> = {
-  __resolveType: TypeResolveFn<'JustReadFeedError' | 'JustReadFeedSuccess', ParentType, ContextType>;
-};
-
-export type JustReadFeedSectionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSection'] = ResolversParentTypes['JustReadFeedSection']> = {
-  items?: Resolver<Array<ResolversTypes['JustReadFeedItem']>, ParentType, ContextType>;
-  layout?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  title?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
-export type JustReadFeedSubscriptionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSubscription'] = ResolversParentTypes['JustReadFeedSubscription']> = {
-  icon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
-  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
-export type JustReadFeedSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['JustReadFeedSuccess'] = ResolversParentTypes['JustReadFeedSuccess']> = {
-  edges?: Resolver<Array<ResolversTypes['JustReadFeedEdge']>, ParentType, ContextType>;
-  pageInfo?: Resolver<ResolversTypes['PageInfo'], ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
 export type LabelResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['Label'] = ResolversParentTypes['Label']> = {
   color?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   createdAt?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
@@ -6466,9 +6466,9 @@ export type QueryResolvers<ContextType = ResolverContext, ParentType extends Res
   getUserPersonalization?: Resolver<ResolversTypes['GetUserPersonalizationResult'], ParentType, ContextType>;
   groups?: Resolver<ResolversTypes['GroupsResult'], ParentType, ContextType>;
   hello?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  home?: Resolver<ResolversTypes['HomeResult'], ParentType, ContextType, Partial<QueryHomeArgs>>;
   integration?: Resolver<ResolversTypes['IntegrationResult'], ParentType, ContextType, RequireFields<QueryIntegrationArgs, 'name'>>;
   integrations?: Resolver<ResolversTypes['IntegrationsResult'], ParentType, ContextType>;
-  justReadFeed?: Resolver<ResolversTypes['JustReadFeedResult'], ParentType, ContextType, Partial<QueryJustReadFeedArgs>>;
   labels?: Resolver<ResolversTypes['LabelsResult'], ParentType, ContextType>;
   me?: Resolver<Maybe<ResolversTypes['User']>, ParentType, ContextType>;
   newsletterEmails?: Resolver<ResolversTypes['NewsletterEmailsResult'], ParentType, ContextType>;
@@ -7650,6 +7650,13 @@ export type Resolvers<ContextType = ResolverContext> = {
   Highlight?: HighlightResolvers<ContextType>;
   HighlightReply?: HighlightReplyResolvers<ContextType>;
   HighlightStats?: HighlightStatsResolvers<ContextType>;
+  HomeEdge?: HomeEdgeResolvers<ContextType>;
+  HomeError?: HomeErrorResolvers<ContextType>;
+  HomeItem?: HomeItemResolvers<ContextType>;
+  HomeResult?: HomeResultResolvers<ContextType>;
+  HomeSection?: HomeSectionResolvers<ContextType>;
+  HomeSubscription?: HomeSubscriptionResolvers<ContextType>;
+  HomeSuccess?: HomeSuccessResolvers<ContextType>;
   ImportFromIntegrationError?: ImportFromIntegrationErrorResolvers<ContextType>;
   ImportFromIntegrationResult?: ImportFromIntegrationResultResolvers<ContextType>;
   ImportFromIntegrationSuccess?: ImportFromIntegrationSuccessResolvers<ContextType>;
@@ -7664,13 +7671,6 @@ export type Resolvers<ContextType = ResolverContext> = {
   JoinGroupError?: JoinGroupErrorResolvers<ContextType>;
   JoinGroupResult?: JoinGroupResultResolvers<ContextType>;
   JoinGroupSuccess?: JoinGroupSuccessResolvers<ContextType>;
-  JustReadFeedEdge?: JustReadFeedEdgeResolvers<ContextType>;
-  JustReadFeedError?: JustReadFeedErrorResolvers<ContextType>;
-  JustReadFeedItem?: JustReadFeedItemResolvers<ContextType>;
-  JustReadFeedResult?: JustReadFeedResultResolvers<ContextType>;
-  JustReadFeedSection?: JustReadFeedSectionResolvers<ContextType>;
-  JustReadFeedSubscription?: JustReadFeedSubscriptionResolvers<ContextType>;
-  JustReadFeedSuccess?: JustReadFeedSuccessResolvers<ContextType>;
   Label?: LabelResolvers<ContextType>;
   LabelsError?: LabelsErrorResolvers<ContextType>;
   LabelsResult?: LabelsResultResolvers<ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index c79c05d23..28e5d7435 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1142,6 +1142,64 @@ enum HighlightType {
   REDACTION
 }
 
+type HomeEdge {
+  cursor: String!
+  node: HomeSection!
+}
+
+type HomeError {
+  errorCodes: [HomeErrorCode!]!
+}
+
+enum HomeErrorCode {
+  BAD_REQUEST
+  PENDING
+  UNAUTHORIZED
+}
+
+type HomeItem {
+  author: String
+  broadcastCount: Int
+  canArchive: Boolean
+  canComment: Boolean
+  canDelete: Boolean
+  canSave: Boolean
+  canShare: Boolean
+  date: Date!
+  dir: String
+  id: ID!
+  likeCount: Int
+  previewContent: String
+  saveCount: Int
+  seen_at: Date
+  subscription: HomeSubscription!
+  thumbnail: String
+  title: String!
+  url: String!
+  wordCount: Int
+}
+
+union HomeResult = HomeError | HomeSuccess
+
+type HomeSection {
+  items: [HomeItem!]!
+  layout: String
+  thumbnail: String
+  title: String
+}
+
+type HomeSubscription {
+  icon: String
+  id: ID!
+  name: String!
+  url: String
+}
+
+type HomeSuccess {
+  edges: [HomeEdge!]!
+  pageInfo: PageInfo!
+}
+
 type ImportFromIntegrationError {
   errorCodes: [ImportFromIntegrationErrorCode!]!
 }
@@ -1228,64 +1286,6 @@ type JoinGroupSuccess {
   group: RecommendationGroup!
 }
 
-type JustReadFeedEdge {
-  cursor: String!
-  node: JustReadFeedSection!
-}
-
-type JustReadFeedError {
-  errorCodes: [JustReadFeedErrorCode!]!
-}
-
-enum JustReadFeedErrorCode {
-  BAD_REQUEST
-  PENDING
-  UNAUTHORIZED
-}
-
-type JustReadFeedItem {
-  author: String
-  broadcastCount: Int
-  canArchive: Boolean
-  canComment: Boolean
-  canDelete: Boolean
-  canSave: Boolean
-  canShare: Boolean
-  date: Date!
-  dir: String
-  id: ID!
-  likeCount: Int
-  previewContent: String
-  saveCount: Int
-  seen_at: Date
-  subscription: JustReadFeedSubscription!
-  thumbnail: String
-  title: String!
-  url: String!
-  wordCount: Int
-}
-
-union JustReadFeedResult = JustReadFeedError | JustReadFeedSuccess
-
-type JustReadFeedSection {
-  items: [JustReadFeedItem!]!
-  layout: String
-  thumbnail: String
-  title: String
-}
-
-type JustReadFeedSubscription {
-  icon: String
-  id: ID!
-  name: String!
-  url: String
-}
-
-type JustReadFeedSuccess {
-  edges: [JustReadFeedEdge!]!
-  pageInfo: PageInfo!
-}
-
 type Label {
   color: String!
   createdAt: Date
@@ -1702,9 +1702,9 @@ type Query {
   getUserPersonalization: GetUserPersonalizationResult!
   groups: GroupsResult!
   hello: String
+  home(after: String, first: Int): HomeResult!
   integration(name: String!): IntegrationResult!
   integrations: IntegrationsResult!
-  justReadFeed(after: String, first: Int): JustReadFeedResult!
   labels: LabelsResult!
   me: User
   newsletterEmails: NewsletterEmailsResult!
diff --git a/packages/api/src/jobs/update_just_read_feed.ts b/packages/api/src/jobs/update_home.ts
similarity index 91%
rename from packages/api/src/jobs/update_just_read_feed.ts
rename to packages/api/src/jobs/update_home.ts
index 5f2cecc59..d83d5e242 100644
--- a/packages/api/src/jobs/update_just_read_feed.ts
+++ b/packages/api/src/jobs/update_home.ts
@@ -3,7 +3,7 @@ import { PublicItem } from '../entity/public_item'
 import { Subscription } from '../entity/subscription'
 import { User } from '../entity/user'
 import { redisDataSource } from '../redis_data_source'
-import { findUnseenPublicItems } from '../services/just_read_feed'
+import { findUnseenPublicItems } from '../services/home'
 import { searchLibraryItems } from '../services/library_item'
 import { Feature, getScores, ScoreApiResponse } from '../services/score'
 import { findSubscriptionsByNames } from '../services/subscriptions'
@@ -11,9 +11,9 @@ import { findActiveUser } from '../services/user'
 import { lanaugeToCode } from '../utils/helpers'
 import { logger } from '../utils/logger'
 
-export const UPDATE_JUST_READ_FEED_JOB = 'UPDATE_JUST_READ_FEED_JOB'
+export const UPDATE_HOME_JOB = 'UPDATE_HOME_JOB'
 
-export interface UpdateJustReadFeedJobData {
+export interface UpdateHomeJobData {
   userId: string
 }
 
@@ -208,7 +208,7 @@ const rankCandidates = async (
 const redisKey = (userId: string) => `just-read-feed:${userId}`
 const MAX_FEED_ITEMS = 500
 
-export const getJustReadFeedSections = async (
+export const getHomeSections = async (
   userId: string,
   limit: number,
   maxScore?: number
@@ -244,7 +244,7 @@ export const getJustReadFeedSections = async (
   return sections
 }
 
-const appendSectionsToFeed = async (
+const appendSectionsToHome = async (
   userId: string,
   sections: Array<Section>
 ) => {
@@ -269,19 +269,19 @@ const appendSectionsToFeed = async (
   pipeline.zremrangebyrank(key, 0, -(MAX_FEED_ITEMS + 1))
   pipeline.zremrangebyscore(key, '-inf', Date.now())
 
-  logger.info('Adding feed sections to redis')
+  logger.info('Adding home sections to redis')
   await pipeline.exec()
 }
 
-const mixFeedItems = (rankedFeedItems: Array<Candidate>): Array<Section> => {
+const mixHomeItems = (rankedHomeItems: Array<Candidate>): Array<Section> => {
   // find the median word count
-  const wordCounts = rankedFeedItems.map((item) => item.wordCount)
+  const wordCounts = rankedHomeItems.map((item) => item.wordCount)
   wordCounts.sort((a, b) => a - b)
   const medianWordCount = wordCounts[Math.floor(wordCounts.length / 2)]
   // separate items into two groups based on word count
   const shortItems: Array<Candidate> = []
   const longItems: Array<Candidate> = []
-  for (const item of rankedFeedItems) {
+  for (const item of rankedHomeItems) {
     if (item.wordCount < medianWordCount) {
       shortItems.push(item)
     } else {
@@ -290,7 +290,7 @@ const mixFeedItems = (rankedFeedItems: Array<Candidate>): Array<Section> => {
   }
   // initialize empty batches
   const batches: Array<Array<Candidate>> = Array.from(
-    { length: Math.floor(rankedFeedItems.length / 10) },
+    { length: Math.floor(rankedHomeItems.length / 10) },
     () => []
   )
 
@@ -363,7 +363,7 @@ const mixFeedItems = (rankedFeedItems: Array<Candidate>): Array<Section> => {
   return sections
 }
 
-export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
+export const updateHome = async (data: UpdateHomeJobData) => {
   const { userId } = data
   const user = await findActiveUser(userId)
   if (!user) {
@@ -371,7 +371,7 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
     return
   }
 
-  logger.info(`Updating just read feed for user ${userId}`)
+  logger.info(`Updating home for user ${userId}`)
 
   const candidates = await selectCandidates(user)
   logger.info(`Found ${candidates.length} candidates`)
@@ -387,11 +387,11 @@ export const updateJustReadFeed = async (data: UpdateJustReadFeedJobData) => {
 
   // TODO: filter candidates
 
-  logger.info('Mix feed items to create sections')
-  const rankedSections = mixFeedItems(rankedCandidates)
+  logger.info('Mix home items to create sections')
+  const rankedSections = mixHomeItems(rankedCandidates)
   logger.info(`Created ${rankedSections.length} sections`)
 
-  logger.info('Appending sections to feed')
-  await appendSectionsToFeed(userId, rankedSections)
-  logger.info('Feed updated for user', { userId })
+  logger.info('Appending sections to home')
+  await appendSectionsToHome(userId, rankedSections)
+  logger.info('Home updated for user', { userId })
 }
diff --git a/packages/api/src/queue-processor.ts b/packages/api/src/queue-processor.ts
index ea7a8fe16..188c37efb 100644
--- a/packages/api/src/queue-processor.ts
+++ b/packages/api/src/queue-processor.ts
@@ -63,10 +63,7 @@ import {
   UPDATE_HIGHLIGHT_JOB,
   UPDATE_LABELS_JOB,
 } from './jobs/update_db'
-import {
-  updateJustReadFeed,
-  UPDATE_JUST_READ_FEED_JOB,
-} from './jobs/update_just_read_feed'
+import { updateHome, UPDATE_HOME_JOB } from './jobs/update_home'
 import { updatePDFContentJob } from './jobs/update_pdf_content'
 import { uploadContentJob, UPLOAD_CONTENT_JOB } from './jobs/upload_content'
 import { redisDataSource } from './redis_data_source'
@@ -193,8 +190,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           return createDigest(job.data)
         case UPLOAD_CONTENT_JOB:
           return uploadContentJob(job.data)
-        case UPDATE_JUST_READ_FEED_JOB:
-          return updateJustReadFeed(job.data)
+        case UPDATE_HOME_JOB:
+          return updateHome(job.data)
         case SCORE_LIBRARY_ITEM_JOB:
           return scoreLibraryItem(job.data)
         default:
diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts
index 13a750a9f..9617db3e7 100644
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@@ -53,6 +53,7 @@ import {
   saveDiscoverArticleResolver,
 } from './discover_feeds'
 import { optInFeatureResolver } from './features'
+import { homeResolver } from './home'
 import { uploadImportFileResolver } from './importers/uploadImportFileResolver'
 import {
   addPopularReadResolver,
@@ -152,7 +153,6 @@ import {
   webhookResolver,
   webhooksResolver,
 } from './index'
-import { justReadFeedResolver } from './just_read_feed'
 import {
   markEmailAsItemResolver,
   recentEmailsResolver,
@@ -360,7 +360,7 @@ export const functionResolvers = {
     feeds: feedsResolver,
     scanFeeds: scanFeedsResolver,
     integration: integrationResolver,
-    justReadFeed: justReadFeedResolver,
+    home: homeResolver,
   },
   User: {
     async intercomHash(
@@ -624,15 +624,13 @@ export const functionResolvers = {
       return newsletterEmail.folder || EXISTING_NEWSLETTER_FOLDER
     },
   },
-  JustReadFeedSection: {
+  HomeSection: {
     async items(
       section: { items: Array<{ id: string }> },
       _: unknown,
       ctx: WithDataSourcesContext
     ) {
-      return ctx.dataLoaders.justReadFeedItems.loadMany(
-        section.items.map((i) => i.id)
-      )
+      return ctx.dataLoaders.homeItems.loadMany(section.items.map((i) => i.id))
     },
   },
   ...resultResolveTypeResolver('Login'),
@@ -734,5 +732,5 @@ export const functionResolvers = {
   ...resultResolveTypeResolver('Integration'),
   ...resultResolveTypeResolver('ExportToIntegration'),
   ...resultResolveTypeResolver('ReplyToEmail'),
-  ...resultResolveTypeResolver('JustReadFeed'),
+  ...resultResolveTypeResolver('Home'),
 }
diff --git a/packages/api/src/resolvers/just_read_feed/index.ts b/packages/api/src/resolvers/home/index.ts
similarity index 52%
rename from packages/api/src/resolvers/just_read_feed/index.ts
rename to packages/api/src/resolvers/home/index.ts
index 1a225dc87..3716fd861 100644
--- a/packages/api/src/resolvers/just_read_feed/index.ts
+++ b/packages/api/src/resolvers/home/index.ts
@@ -1,66 +1,57 @@
 import {
-  JustReadFeedError,
-  JustReadFeedErrorCode,
-  JustReadFeedItem,
-  JustReadFeedSection,
-  JustReadFeedSuccess,
-  QueryJustReadFeedArgs,
+  HomeError,
+  HomeErrorCode,
+  HomeItem,
+  HomeSection,
+  HomeSuccess,
+  QueryHomeArgs,
 } from '../../generated/graphql'
-import { getJustReadFeedSections } from '../../jobs/update_just_read_feed'
+import { getHomeSections } from '../../jobs/update_home'
 import { getJob } from '../../queue-processor'
 import { Merge } from '../../util'
-import {
-  enqueueUpdateJustReadFeed,
-  updateJustReadFeedJobId,
-} from '../../utils/createTask'
+import { enqueueUpdateHomeJob, updateHomeJobId } from '../../utils/createTask'
 import { authorized } from '../../utils/gql-utils'
 
-type PartialJustReadFeedItem = Merge<
-  Partial<JustReadFeedItem>,
-  { type: string }
->
-type PartialJustReadFeedSection = Merge<
-  JustReadFeedSection,
-  { items: Array<PartialJustReadFeedItem> }
->
-type PartialJustReadFeedSuccess = Merge<
-  JustReadFeedSuccess,
+type PartialHomeItem = Merge<Partial<HomeItem>, { type: string }>
+type PartialHomeSection = Merge<HomeSection, { items: Array<PartialHomeItem> }>
+type PartialHomeSuccess = Merge<
+  HomeSuccess,
   {
-    edges: Array<{ cursor: string; node: PartialJustReadFeedSection }>
+    edges: Array<{ cursor: string; node: PartialHomeSection }>
   }
 >
 // This resolver is used to fetch the just read feed for the user.
 // when the feed is empty, it enqueues a job to update the feed.
 // when client tries to fetch more then the feed has, it enqueues a job to update the feed.
-export const justReadFeedResolver = authorized<
-  PartialJustReadFeedSuccess,
-  JustReadFeedError,
-  QueryJustReadFeedArgs
+export const homeResolver = authorized<
+  PartialHomeSuccess,
+  HomeError,
+  QueryHomeArgs
 >(async (_, { first, after }, { uid, log }) => {
   const limit = first || 10
   const cursor = after ? parseInt(after) : undefined
 
-  const sections = await getJustReadFeedSections(uid, limit, cursor)
+  const sections = await getHomeSections(uid, limit, cursor)
   log.info('Just read feed sections fetched')
 
   if (sections.length === 0) {
-    const existingJob = await getJob(updateJustReadFeedJobId(uid))
+    const existingJob = await getJob(updateHomeJobId(uid))
     if (existingJob) {
       log.info('Just read feed update job already enqueued')
 
       return {
-        errorCodes: [JustReadFeedErrorCode.Pending],
+        errorCodes: [HomeErrorCode.Pending],
       }
     }
 
-    await enqueueUpdateJustReadFeed({
+    await enqueueUpdateHomeJob({
       userId: uid,
     })
 
     log.info('Just read feed update enqueued')
 
     return {
-      errorCodes: [JustReadFeedErrorCode.Pending],
+      errorCodes: [HomeErrorCode.Pending],
     }
   }
 
diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts
index dacc50476..3c6d76f25 100644
--- a/packages/api/src/resolvers/types.ts
+++ b/packages/api/src/resolvers/types.ts
@@ -10,7 +10,7 @@ import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
 import { Recommendation } from '../entity/recommendation'
 import { UploadFile } from '../entity/upload_file'
-import { JustReadFeedItem } from '../generated/graphql'
+import { HomeItem } from '../generated/graphql'
 import { PubsubClient } from '../pubsub'
 
 export interface Claims {
@@ -52,7 +52,7 @@ export interface RequestContext {
     highlights: DataLoader<string, Highlight[]>
     recommendations: DataLoader<string, Recommendation[]>
     uploadFiles: DataLoader<string, UploadFile | undefined>
-    justReadFeedItems: DataLoader<string, JustReadFeedItem>
+    homeItems: DataLoader<string, HomeItem>
   }
 }
 
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index c0b9345cb..8ea86abab 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3101,14 +3101,14 @@ const schema = gql`
     SUBSCRIBE
   }
 
-  type JustReadFeedSubscription {
+  type HomeSubscription {
     id: ID!
     name: String!
     url: String
     icon: String
   }
 
-  type JustReadFeedItem {
+  type HomeItem {
     id: ID!
     title: String!
     url: String!
@@ -3122,7 +3122,7 @@ const schema = gql`
     dir: String
     seen_at: Date
     wordCount: Int
-    subscription: JustReadFeedSubscription!
+    subscription: HomeSubscription!
     canSave: Boolean
     canComment: Boolean
     canShare: Boolean
@@ -3130,34 +3130,34 @@ const schema = gql`
     canDelete: Boolean
   }
 
-  type JustReadFeedSection {
+  type HomeSection {
     title: String
     layout: String
-    items: [JustReadFeedItem!]!
+    items: [HomeItem!]!
     thumbnail: String
   }
 
-  type JustReadFeedEdge {
+  type HomeEdge {
     cursor: String!
-    node: JustReadFeedSection!
+    node: HomeSection!
   }
 
-  type JustReadFeedSuccess {
-    edges: [JustReadFeedEdge!]!
+  type HomeSuccess {
+    edges: [HomeEdge!]!
     pageInfo: PageInfo!
   }
 
-  enum JustReadFeedErrorCode {
+  enum HomeErrorCode {
     UNAUTHORIZED
     BAD_REQUEST
     PENDING
   }
 
-  type JustReadFeedError {
-    errorCodes: [JustReadFeedErrorCode!]!
+  type HomeError {
+    errorCodes: [HomeErrorCode!]!
   }
 
-  union JustReadFeedResult = JustReadFeedSuccess | JustReadFeedError
+  union HomeResult = HomeSuccess | HomeError
 
   type SubscriptionRootType {
     hello: String # for testing only
@@ -3358,7 +3358,7 @@ const schema = gql`
     feeds(input: FeedsInput!): FeedsResult!
     discoverFeeds: DiscoverFeedResult!
     scanFeeds(input: ScanFeedsInput!): ScanFeedsResult!
-    justReadFeed(first: Int, after: String): JustReadFeedResult!
+    home(first: Int, after: String): HomeResult!
   }
 
   schema {
diff --git a/packages/api/src/services/just_read_feed.ts b/packages/api/src/services/home.ts
similarity index 92%
rename from packages/api/src/services/just_read_feed.ts
rename to packages/api/src/services/home.ts
index a38da1650..f53c07eee 100644
--- a/packages/api/src/services/just_read_feed.ts
+++ b/packages/api/src/services/home.ts
@@ -1,12 +1,12 @@
 import { PublicItem } from '../entity/public_item'
-import { JustReadFeedItem } from '../generated/graphql'
+import { HomeItem } from '../generated/graphql'
 import { authTrx } from '../repository'
 import { findLibraryItemsByIds } from './library_item'
 
-export const batchGetJustReadFeedItems = async (
+export const batchGetHomeItems = async (
   ids: readonly string[]
-): Promise<Array<JustReadFeedItem>> => {
-  const libraryItems = await findLibraryItemsByIds(ids as string[], '')
+): Promise<Array<HomeItem>> => {
+  const libraryItems = await findLibraryItemsByIds(ids as string[])
 
   const publicItems = await authTrx(async (tx) =>
     tx
@@ -57,7 +57,7 @@ export const batchGetJustReadFeedItems = async (
           : undefined
       }
     })
-    .filter((item) => item !== undefined) as JustReadFeedItem[]
+    .filter((item) => item !== undefined) as Array<HomeItem>
 }
 
 export const findUnseenPublicItems = async (
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index 03a66e97c..d24ce4dff 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -775,7 +775,7 @@ export const findRecentLibraryItems = async (
 
 export const findLibraryItemsByIds = async (
   ids: string[],
-  userId: string,
+  userId?: string,
   options?: {
     select?: (keyof LibraryItem)[]
   }
diff --git a/packages/api/src/utils/createTask.ts b/packages/api/src/utils/createTask.ts
index 2ba80e4be..26ef6362b 100644
--- a/packages/api/src/utils/createTask.ts
+++ b/packages/api/src/utils/createTask.ts
@@ -57,10 +57,7 @@ import {
   UPDATE_HIGHLIGHT_JOB,
   UPDATE_LABELS_JOB,
 } from '../jobs/update_db'
-import {
-  UpdateJustReadFeedJobData,
-  UPDATE_JUST_READ_FEED_JOB,
-} from '../jobs/update_just_read_feed'
+import { UpdateHomeJobData, UPDATE_HOME_JOB } from '../jobs/update_home'
 import {
   UploadContentJobData,
   UPLOAD_CONTENT_JOB,
@@ -93,7 +90,7 @@ export const getJobPriority = (jobName: string): number => {
     case UPDATE_HIGHLIGHT_JOB:
     case SYNC_READ_POSITIONS_JOB_NAME:
     case SEND_EMAIL_JOB:
-    case UPDATE_JUST_READ_FEED_JOB:
+    case UPDATE_HOME_JOB:
       return 1
     case TRIGGER_RULE_JOB_NAME:
     case CALL_WEBHOOK_JOB_NAME:
@@ -991,22 +988,20 @@ export const enqueueBulkUploadContentJob = async (
   return queue.addBulk(jobs)
 }
 
-export const updateJustReadFeedJobId = (userId: string) =>
-  `${UPDATE_JUST_READ_FEED_JOB}_${userId}_${JOB_VERSION}`
+export const updateHomeJobId = (userId: string) =>
+  `${UPDATE_HOME_JOB}_${userId}_${JOB_VERSION}`
 
-export const enqueueUpdateJustReadFeed = async (
-  data: UpdateJustReadFeedJobData
-) => {
+export const enqueueUpdateHomeJob = async (data: UpdateHomeJobData) => {
   const queue = await getBackendQueue()
   if (!queue) {
     return undefined
   }
 
-  return queue.add(UPDATE_JUST_READ_FEED_JOB, data, {
-    jobId: updateJustReadFeedJobId(data.userId),
+  return queue.add(UPDATE_HOME_JOB, data, {
+    jobId: updateHomeJobId(data.userId),
     removeOnComplete: true,
     removeOnFail: true,
-    priority: getJobPriority(UPDATE_JUST_READ_FEED_JOB),
+    priority: getJobPriority(UPDATE_HOME_JOB),
     attempts: 3,
   })
 }

From 57b28c39bf23126ebd92a68c8f7842ed41e95b50 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 21:17:54 +0800
Subject: [PATCH 27/33] store topic as a ltree

---
 packages/api/src/entity/library_item.ts          | 2 +-
 packages/db/migrations/0177.do.public_item.sql   | 9 ++++++---
 packages/db/migrations/0177.undo.public_item.sql | 2 ++
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/entity/library_item.ts b/packages/api/src/entity/library_item.ts
index 8168b8548..8fb0edc94 100644
--- a/packages/api/src/entity/library_item.ts
+++ b/packages/api/src/entity/library_item.ts
@@ -208,7 +208,7 @@ export class LibraryItem {
   @Column('timestamptz')
   seenAt?: Date
 
-  @Column('text')
+  @Column('ltree')
   topic?: string
 
   @Column('timestamptz')
diff --git a/packages/db/migrations/0177.do.public_item.sql b/packages/db/migrations/0177.do.public_item.sql
index c4b465845..ded66eefe 100755
--- a/packages/db/migrations/0177.do.public_item.sql
+++ b/packages/db/migrations/0177.do.public_item.sql
@@ -79,11 +79,14 @@ CREATE INDEX public_item_interaction_public_item_id_idx ON omnivore.public_item_
 CREATE TRIGGER update_public_item_interactions_modtime BEFORE UPDATE ON omnivore.public_item_interactions FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column();
 GRANT SELECT, INSERT, UPDATE ON omnivore.public_item_interactions TO omnivore_user;
 
+CREATE EXTENSION LTREE;
 
 ALTER TABLE omnivore.library_item 
-    ADD COLUMN seen_at timestamptz,
-    ADD COLUMN digested_at timestamptz,
-    ADD COLUMN topic TEXT,
+    ADD COLUMN seen_at TIMESTAMPTZ,
+    ADD COLUMN digested_at TIMESTAMPTZ,
+    ADD COLUMN topic LTREE,
     ADD COLUMN score FLOAT;
 
+CREATE INDEX library_item_topic_idx ON omnivore.library_item USING GIST (topic);
+
 COMMIT;
diff --git a/packages/db/migrations/0177.undo.public_item.sql b/packages/db/migrations/0177.undo.public_item.sql
index 314dcccf2..16e0e0d40 100755
--- a/packages/db/migrations/0177.undo.public_item.sql
+++ b/packages/db/migrations/0177.undo.public_item.sql
@@ -9,6 +9,8 @@ DROP TABLE omnivore.public_item_stats;
 DROP TABLE omnivore.public_item;
 DROP TABLE omnivore.public_item_source;
 
+DROP INDEX omnivore.library_item_topic_idx;
+
 ALTER TABLE omnivore.library_item 
     DROP COLUMN seen_at,
     DROP COLUMN digested_at,

From 1c07e41efd690baa0e641034eebd4221c4c05faa Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 21:20:49 +0800
Subject: [PATCH 28/33] update comment

---
 packages/api/src/jobs/update_home.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts
index d83d5e242..70d54f097 100644
--- a/packages/api/src/jobs/update_home.ts
+++ b/packages/api/src/jobs/update_home.ts
@@ -147,7 +147,7 @@ const selectCandidates = async (user: User): Promise<Array<Candidate>> => {
   const publicCandidatesSize = publicCandidates.length
   logger.info(`Found ${publicCandidatesSize} public candidates`)
 
-  // return 100 candidates and 70 from private and 30 from public
+  // returns 100 candidates which are a mix of private and public candidates
   return [...privateCandidates, ...publicCandidates]
 }
 

From 88dcdbfb2d12e1cc0cdee9f9708734e29ff25c48 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Mon, 27 May 2024 21:29:15 +0800
Subject: [PATCH 29/33] update score api client

---
 packages/api/src/jobs/score_library_item.ts | 3 ++-
 packages/api/src/jobs/update_home.ts        | 3 ++-
 packages/api/src/services/score.ts          | 5 +++--
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/packages/api/src/jobs/score_library_item.ts b/packages/api/src/jobs/score_library_item.ts
index 3e001bf45..75f71f9dc 100644
--- a/packages/api/src/jobs/score_library_item.ts
+++ b/packages/api/src/jobs/score_library_item.ts
@@ -42,6 +42,7 @@ export const scoreLibraryItem = async (
 
   const itemFeatures = {
     [libraryItem.id]: {
+      library_item_id: libraryItem.id,
       title: libraryItem.title,
       has_thumbnail: !!libraryItem.thumbnail,
       has_site_icon: !!libraryItem.siteIcon,
@@ -59,7 +60,7 @@ export const scoreLibraryItem = async (
 
   const scores = await getScores({
     user_id: userId,
-    item_features: itemFeatures,
+    items: itemFeatures,
   })
 
   logger.info('Scores', scores)
diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts
index 70d54f097..4189def79 100644
--- a/packages/api/src/jobs/update_home.ts
+++ b/packages/api/src/jobs/update_home.ts
@@ -166,8 +166,9 @@ const rankCandidates = async (
 
   const data = {
     user_id: userId,
-    item_features: unscoredCandidates.reduce((acc, item) => {
+    items: unscoredCandidates.reduce((acc, item) => {
       acc[item.id] = {
+        library_item_id: item.id,
         title: item.title,
         has_thumbnail: !!item.thumbnail,
         has_site_icon: !!item.siteIcon,
diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
index 4a4bbf6d5..9d41fead2 100644
--- a/packages/api/src/services/score.ts
+++ b/packages/api/src/services/score.ts
@@ -1,4 +1,5 @@
 export interface Feature {
+  library_item_id?: string
   title: string
   has_thumbnail: boolean
   has_site_icon: boolean
@@ -15,7 +16,7 @@ export interface Feature {
 
 export interface ScoreApiRequestBody {
   user_id: string
-  item_features: Record<string, Feature> // item_id -> feature
+  items: Record<string, Feature> // item_id -> feature
 }
 
 export type ScoreApiResponse = Record<string, number> // item_id -> score
@@ -23,7 +24,7 @@ export type ScoreApiResponse = Record<string, number> // item_id -> score
 export const getScores = async (
   data: ScoreApiRequestBody
 ): Promise<ScoreApiResponse> => {
-  const API_URL = 'http://127.0.0.1:5000/predictions'
+  const API_URL = 'http://digest-score/batch'
   // const token = process.env.SCORE_API_TOKEN
 
   // if (!token) {

From e131afc183de1051ab0a042659a7e2bd70eaf7fd Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 28 May 2024 14:45:33 +0800
Subject: [PATCH 30/33] update API

---
 packages/api/src/apollo.ts                    |  6 +-
 packages/api/src/generated/graphql.ts         | 52 +++++++-----
 packages/api/src/generated/schema.graphql     | 24 ++++--
 packages/api/src/jobs/update_home.ts          |  5 +-
 .../api/src/resolvers/function_resolvers.ts   | 84 ++++++++++++++++++-
 packages/api/src/resolvers/home/index.ts      |  2 +-
 packages/api/src/resolvers/types.ts           |  5 +-
 packages/api/src/schema.ts                    | 14 +++-
 packages/api/src/services/home.ts             | 20 ++++-
 packages/api/src/services/library_item.ts     |  4 +
 .../db/migrations/0177.undo.public_item.sql   |  2 +
 11 files changed, 175 insertions(+), 43 deletions(-)

diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts
index d4885fa7e..d929d5c64 100644
--- a/packages/api/src/apollo.ts
+++ b/packages/api/src/apollo.ts
@@ -32,8 +32,9 @@ import { ClaimsToSet, RequestContext, ResolverContext } from './resolvers/types'
 import ScalarResolvers from './scalars'
 import typeDefs from './schema'
 import { batchGetHighlightsFromLibraryItemIds } from './services/highlights'
-import { batchGetHomeItems } from './services/home'
+import { batchGetPublicItems } from './services/home'
 import { batchGetLabelsFromLibraryItemIds } from './services/labels'
+import { batchGetLibraryItems } from './services/library_item'
 import { batchGetRecommendationsFromLibraryItemIds } from './services/recommendation'
 import {
   countDailyServiceUsage,
@@ -113,7 +114,8 @@ const contextFunc: ContextFunction<ExpressContext, ResolverContext> = async ({
         batchGetRecommendationsFromLibraryItemIds
       ),
       uploadFiles: new DataLoader(batchGetUploadFilesByIds),
-      homeItems: new DataLoader(batchGetHomeItems),
+      libraryItems: new DataLoader(batchGetLibraryItems),
+      publicItems: new DataLoader(batchGetPublicItems),
     },
   }
 
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index a3f839346..6a80b3f07 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1303,13 +1303,29 @@ export type HomeItem = {
   previewContent?: Maybe<Scalars['String']>;
   saveCount?: Maybe<Scalars['Int']>;
   seen_at?: Maybe<Scalars['Date']>;
-  subscription: HomeSubscription;
+  subscription?: Maybe<HomeItemSource>;
   thumbnail?: Maybe<Scalars['String']>;
   title: Scalars['String'];
   url: Scalars['String'];
   wordCount?: Maybe<Scalars['Int']>;
 };
 
+export type HomeItemSource = {
+  __typename?: 'HomeItemSource';
+  icon?: Maybe<Scalars['String']>;
+  id?: Maybe<Scalars['ID']>;
+  name: Scalars['String'];
+  type: HomeItemSourceType;
+  url?: Maybe<Scalars['String']>;
+};
+
+export enum HomeItemSourceType {
+  Library = 'LIBRARY',
+  Newsletter = 'NEWSLETTER',
+  Recommendation = 'RECOMMENDATION',
+  Rss = 'RSS'
+}
+
 export type HomeResult = HomeError | HomeSuccess;
 
 export type HomeSection = {
@@ -1320,14 +1336,6 @@ export type HomeSection = {
   title?: Maybe<Scalars['String']>;
 };
 
-export type HomeSubscription = {
-  __typename?: 'HomeSubscription';
-  icon?: Maybe<Scalars['String']>;
-  id: Scalars['ID'];
-  name: Scalars['String'];
-  url?: Maybe<Scalars['String']>;
-};
-
 export type HomeSuccess = {
   __typename?: 'HomeSuccess';
   edges: Array<HomeEdge>;
@@ -4259,9 +4267,10 @@ export type ResolversTypes = {
   HomeError: ResolverTypeWrapper<HomeError>;
   HomeErrorCode: HomeErrorCode;
   HomeItem: ResolverTypeWrapper<HomeItem>;
+  HomeItemSource: ResolverTypeWrapper<HomeItemSource>;
+  HomeItemSourceType: HomeItemSourceType;
   HomeResult: ResolversTypes['HomeError'] | ResolversTypes['HomeSuccess'];
   HomeSection: ResolverTypeWrapper<HomeSection>;
-  HomeSubscription: ResolverTypeWrapper<HomeSubscription>;
   HomeSuccess: ResolverTypeWrapper<HomeSuccess>;
   ID: ResolverTypeWrapper<Scalars['ID']>;
   ImportFromIntegrationError: ResolverTypeWrapper<ImportFromIntegrationError>;
@@ -4815,9 +4824,9 @@ export type ResolversParentTypes = {
   HomeEdge: HomeEdge;
   HomeError: HomeError;
   HomeItem: HomeItem;
+  HomeItemSource: HomeItemSource;
   HomeResult: ResolversParentTypes['HomeError'] | ResolversParentTypes['HomeSuccess'];
   HomeSection: HomeSection;
-  HomeSubscription: HomeSubscription;
   HomeSuccess: HomeSuccess;
   ID: Scalars['ID'];
   ImportFromIntegrationError: ImportFromIntegrationError;
@@ -6034,7 +6043,7 @@ export type HomeItemResolvers<ContextType = ResolverContext, ParentType extends
   previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   saveCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
-  subscription?: Resolver<ResolversTypes['HomeSubscription'], ParentType, ContextType>;
+  subscription?: Resolver<Maybe<ResolversTypes['HomeItemSource']>, ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
@@ -6042,6 +6051,15 @@ export type HomeItemResolvers<ContextType = ResolverContext, ParentType extends
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
+export type HomeItemSourceResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeItemSource'] = ResolversParentTypes['HomeItemSource']> = {
+  icon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  id?: Resolver<Maybe<ResolversTypes['ID']>, ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  type?: Resolver<ResolversTypes['HomeItemSourceType'], ParentType, ContextType>;
+  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type HomeResultResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeResult'] = ResolversParentTypes['HomeResult']> = {
   __resolveType: TypeResolveFn<'HomeError' | 'HomeSuccess', ParentType, ContextType>;
 };
@@ -6054,14 +6072,6 @@ export type HomeSectionResolvers<ContextType = ResolverContext, ParentType exten
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 };
 
-export type HomeSubscriptionResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeSubscription'] = ResolversParentTypes['HomeSubscription']> = {
-  icon?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  id?: Resolver<ResolversTypes['ID'], ParentType, ContextType>;
-  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
 export type HomeSuccessResolvers<ContextType = ResolverContext, ParentType extends ResolversParentTypes['HomeSuccess'] = ResolversParentTypes['HomeSuccess']> = {
   edges?: Resolver<Array<ResolversTypes['HomeEdge']>, ParentType, ContextType>;
   pageInfo?: Resolver<ResolversTypes['PageInfo'], ParentType, ContextType>;
@@ -7653,9 +7663,9 @@ export type Resolvers<ContextType = ResolverContext> = {
   HomeEdge?: HomeEdgeResolvers<ContextType>;
   HomeError?: HomeErrorResolvers<ContextType>;
   HomeItem?: HomeItemResolvers<ContextType>;
+  HomeItemSource?: HomeItemSourceResolvers<ContextType>;
   HomeResult?: HomeResultResolvers<ContextType>;
   HomeSection?: HomeSectionResolvers<ContextType>;
-  HomeSubscription?: HomeSubscriptionResolvers<ContextType>;
   HomeSuccess?: HomeSuccessResolvers<ContextType>;
   ImportFromIntegrationError?: ImportFromIntegrationErrorResolvers<ContextType>;
   ImportFromIntegrationResult?: ImportFromIntegrationResultResolvers<ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 28e5d7435..2b09a639f 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1172,13 +1172,28 @@ type HomeItem {
   previewContent: String
   saveCount: Int
   seen_at: Date
-  subscription: HomeSubscription!
+  subscription: HomeItemSource
   thumbnail: String
   title: String!
   url: String!
   wordCount: Int
 }
 
+type HomeItemSource {
+  icon: String
+  id: ID
+  name: String!
+  type: HomeItemSourceType!
+  url: String
+}
+
+enum HomeItemSourceType {
+  LIBRARY
+  NEWSLETTER
+  RECOMMENDATION
+  RSS
+}
+
 union HomeResult = HomeError | HomeSuccess
 
 type HomeSection {
@@ -1188,13 +1203,6 @@ type HomeSection {
   title: String
 }
 
-type HomeSubscription {
-  icon: String
-  id: ID!
-  name: String!
-  url: String
-}
-
 type HomeSuccess {
   edges: [HomeEdge!]!
   pageInfo: PageInfo!
diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts
index 4189def79..26561795b 100644
--- a/packages/api/src/jobs/update_home.ts
+++ b/packages/api/src/jobs/update_home.ts
@@ -66,7 +66,7 @@ const libraryItemToCandidate = (
   dir: item.directionality || 'ltr',
   date: item.createdAt,
   topic: item.topic,
-  wordCount: item.wordCount!,
+  wordCount: item.wordCount || 0,
   siteName: item.siteName || undefined,
   siteIcon: item.siteIcon || undefined,
   folder: item.folder,
@@ -91,8 +91,9 @@ const publicItemToCandidate = (item: PublicItem): Candidate => ({
   dir: item.dir || 'ltr',
   date: item.createdAt,
   topic: item.topic,
-  wordCount: item.wordCount!,
+  wordCount: item.wordCount || 0,
   siteIcon: item.siteIcon,
+  siteName: item.siteName,
   publishedAt: item.publishedAt,
   subscription: {
     name: item.source.name,
diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts
index 9617db3e7..213a774b7 100644
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@@ -4,11 +4,14 @@
 /* eslint-disable @typescript-eslint/no-unsafe-member-access */
 /* eslint-disable @typescript-eslint/explicit-module-boundary-types */
 import { createHmac } from 'crypto'
+import { isError } from 'lodash'
 import { Highlight as HighlightEntity } from '../entity/highlight'
+import { LibraryItem } from '../entity/library_item'
 import {
   EXISTING_NEWSLETTER_FOLDER,
   NewsletterEmail,
 } from '../entity/newsletter_email'
+import { PublicItem } from '../entity/public_item'
 import {
   DEFAULT_SUBSCRIPTION_FOLDER,
   Subscription,
@@ -17,6 +20,7 @@ import { env } from '../env'
 import {
   Article,
   Highlight,
+  HomeItem,
   Label,
   PageType,
   Recommendation,
@@ -626,11 +630,87 @@ export const functionResolvers = {
   },
   HomeSection: {
     async items(
-      section: { items: Array<{ id: string }> },
+      section: {
+        items: Array<{ id: string; type: 'library_item' | 'public_item' }>
+      },
       _: unknown,
       ctx: WithDataSourcesContext
     ) {
-      return ctx.dataLoaders.homeItems.loadMany(section.items.map((i) => i.id))
+      const libraryItemIds = section.items
+        .filter((item) => item.type === 'library_item')
+        .map((item) => item.id)
+      const libraryItems = (
+        await ctx.dataLoaders.libraryItems.loadMany(libraryItemIds)
+      ).filter((libraryItem) => !isError(libraryItem)) as Array<LibraryItem>
+
+      const publicItemIds = section.items
+        .filter((item) => item.type === 'public_item')
+        .map((item) => item.id)
+      const publicItems = (
+        await ctx.dataLoaders.publicItems.loadMany(publicItemIds)
+      ).filter((publicItem) => !isError(publicItem)) as Array<PublicItem>
+
+      return libraryItems
+        .map(
+          (libraryItem) =>
+            ({
+              id: libraryItem.id,
+              title: libraryItem.title,
+              author: libraryItem.author,
+              thumbnail: libraryItem.thumbnail,
+              wordCount: libraryItem.wordCount,
+              date: libraryItem.savedAt,
+              url: libraryItem.originalUrl,
+              canArchive: !libraryItem.archivedAt,
+              canDelete: !libraryItem.deletedAt,
+              canSave: false,
+              dir: libraryItem.directionality,
+              subscription: null,
+              previewContent: libraryItem.description,
+            } as HomeItem)
+        )
+        .concat(
+          publicItems.map(
+            (publicItem) =>
+              ({
+                id: publicItem.id,
+                title: publicItem.title,
+                author: publicItem.author,
+                dir: publicItem.dir,
+                previewContent: publicItem.previewContent,
+                thumbnail: publicItem.thumbnail,
+                wordCount: publicItem.wordCount,
+                date: publicItem.createdAt,
+                url: publicItem.url,
+                canArchive: false,
+                canDelete: false,
+                canSave: true,
+                broadcastCount: publicItem.stats.broadcastCount,
+                likeCount: publicItem.stats.likeCount,
+                saveCount: publicItem.stats.saveCount,
+                subscription: publicItem.source,
+              } as HomeItem)
+          )
+        )
+    },
+  },
+  HomeItem: {
+    async subscription(
+      item: HomeItem,
+      _: unknown,
+      ctx: WithDataSourcesContext
+    ) {
+      // if (item.subscription) {
+      //   return item.subscription
+      // }
+      // const subscription = await ctx.dataLoaders.subscriptions.load(item.id)
+      // if (!subscription) {
+      //   return {
+      //     name: item.siteName,
+      //     icon: item.siteIcon,
+      //     type: 'rss',
+      //   }
+      // }
     },
   },
   ...resultResolveTypeResolver('Login'),
diff --git a/packages/api/src/resolvers/home/index.ts b/packages/api/src/resolvers/home/index.ts
index 3716fd861..a153fd5c0 100644
--- a/packages/api/src/resolvers/home/index.ts
+++ b/packages/api/src/resolvers/home/index.ts
@@ -28,7 +28,7 @@ export const homeResolver = authorized<
   HomeError,
   QueryHomeArgs
 >(async (_, { first, after }, { uid, log }) => {
-  const limit = first || 10
+  const limit = first || 6
   const cursor = after ? parseInt(after) : undefined
 
   const sections = await getHomeSections(uid, limit, cursor)
diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts
index 3c6d76f25..ef6adfeff 100644
--- a/packages/api/src/resolvers/types.ts
+++ b/packages/api/src/resolvers/types.ts
@@ -8,6 +8,8 @@ import winston from 'winston'
 import { ReadingProgressDataSource } from '../datasources/reading_progress_data_source'
 import { Highlight } from '../entity/highlight'
 import { Label } from '../entity/label'
+import { LibraryItem } from '../entity/library_item'
+import { PublicItem } from '../entity/public_item'
 import { Recommendation } from '../entity/recommendation'
 import { UploadFile } from '../entity/upload_file'
 import { HomeItem } from '../generated/graphql'
@@ -52,7 +54,8 @@ export interface RequestContext {
     highlights: DataLoader<string, Highlight[]>
     recommendations: DataLoader<string, Recommendation[]>
     uploadFiles: DataLoader<string, UploadFile | undefined>
-    homeItems: DataLoader<string, HomeItem>
+    libraryItems: DataLoader<string, LibraryItem>
+    publicItems: DataLoader<string, PublicItem>
   }
 }
 
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index 8ea86abab..ed74e5509 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3101,11 +3101,19 @@ const schema = gql`
     SUBSCRIBE
   }
 
-  type HomeSubscription {
-    id: ID!
+  enum HomeItemSourceType {
+    RSS
+    NEWSLETTER
+    RECOMMENDATION
+    LIBRARY
+  }
+
+  type HomeItemSource {
+    id: ID
     name: String!
     url: String
     icon: String
+    type: HomeItemSourceType!
   }
 
   type HomeItem {
@@ -3122,7 +3130,7 @@ const schema = gql`
     dir: String
     seen_at: Date
     wordCount: Int
-    subscription: HomeSubscription!
+    subscription: HomeItemSource
     canSave: Boolean
     canComment: Boolean
     canShare: Boolean
diff --git a/packages/api/src/services/home.ts b/packages/api/src/services/home.ts
index f53c07eee..c70a98003 100644
--- a/packages/api/src/services/home.ts
+++ b/packages/api/src/services/home.ts
@@ -3,6 +3,18 @@ import { HomeItem } from '../generated/graphql'
 import { authTrx } from '../repository'
 import { findLibraryItemsByIds } from './library_item'
 
+export const batchGetPublicItems = async (
+  ids: readonly string[]
+): Promise<Array<PublicItem>> => {
+  return authTrx(async (tx) =>
+    tx
+      .getRepository(PublicItem)
+      .createQueryBuilder('public_item')
+      .where('public_item.id IN (:...ids)', { ids })
+      .getMany()
+  )
+}
+
 export const batchGetHomeItems = async (
   ids: readonly string[]
 ): Promise<Array<HomeItem>> => {
@@ -38,11 +50,13 @@ export const batchGetHomeItems = async (
           canDelete: !libraryItem.deletedAt,
           canSave: false,
           dir: libraryItem.directionality,
-        }
+          subscription: null,
+          previewContent: libraryItem.description,
+        } as HomeItem
       } else {
         const publicItem = publicItems.find((pi) => pi.id === id)
         return publicItem
-          ? {
+          ? ({
               ...publicItem,
               date: publicItem.createdAt,
               url: publicItem.url,
@@ -53,7 +67,7 @@ export const batchGetHomeItems = async (
               likeCount: publicItem.stats.likeCount,
               saveCount: publicItem.stats.saveCount,
               subscription: publicItem.source,
-            }
+            } as HomeItem)
           : undefined
       }
     })
diff --git a/packages/api/src/services/library_item.ts b/packages/api/src/services/library_item.ts
index d24ce4dff..b90aa0e80 100644
--- a/packages/api/src/services/library_item.ts
+++ b/packages/api/src/services/library_item.ts
@@ -140,6 +140,10 @@ interface Select {
 
 const readingProgressDataSource = new ReadingProgressDataSource()
 
+export const batchGetLibraryItems = async (ids: readonly string[]) => {
+  return findLibraryItemsByIds(ids as string[])
+}
+
 export const getItemUrl = (id: string) => `${env.client.url}/me/${id}`
 
 const markItemAsRead = async (libraryItemId: string, userId: string) => {
diff --git a/packages/db/migrations/0177.undo.public_item.sql b/packages/db/migrations/0177.undo.public_item.sql
index 16e0e0d40..cb068a36c 100755
--- a/packages/db/migrations/0177.undo.public_item.sql
+++ b/packages/db/migrations/0177.undo.public_item.sql
@@ -17,4 +17,6 @@ ALTER TABLE omnivore.library_item
     DROP COLUMN topic,
     DROP COLUMN score;
 
+DROP EXTENSION LTREE;
+
 COMMIT;

From 6b835f0872cf3a913071e749def8a919174770b9 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 28 May 2024 15:02:22 +0800
Subject: [PATCH 31/33] batch load sources

---
 packages/api/src/apollo.ts                    |  8 +++
 packages/api/src/generated/graphql.ts         |  4 +-
 packages/api/src/generated/schema.graphql     |  2 +-
 .../api/src/resolvers/function_resolvers.ts   | 52 +++++++++++++------
 packages/api/src/resolvers/types.ts           |  2 +
 packages/api/src/schema.ts                    |  2 +-
 6 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/packages/api/src/apollo.ts b/packages/api/src/apollo.ts
index d929d5c64..67eb0b65a 100644
--- a/packages/api/src/apollo.ts
+++ b/packages/api/src/apollo.ts
@@ -40,6 +40,7 @@ import {
   countDailyServiceUsage,
   createServiceUsage,
 } from './services/service_usage'
+import { findSubscriptionsByNames } from './services/subscriptions'
 import { batchGetUploadFilesByIds } from './services/upload_file'
 import { tracer } from './tracing'
 import { getClaimsByToken, setAuthInCookie } from './utils/auth'
@@ -116,6 +117,13 @@ const contextFunc: ContextFunction<ExpressContext, ResolverContext> = async ({
       uploadFiles: new DataLoader(batchGetUploadFilesByIds),
       libraryItems: new DataLoader(batchGetLibraryItems),
       publicItems: new DataLoader(batchGetPublicItems),
+      subscriptions: new DataLoader(async (names: readonly string[]) => {
+        if (!claims?.uid) {
+          throw new Error('No user id found in claims')
+        }
+
+        return findSubscriptionsByNames(claims?.uid || '', names as string[])
+      }),
     },
   }
 
diff --git a/packages/api/src/generated/graphql.ts b/packages/api/src/generated/graphql.ts
index 6a80b3f07..ee60e84b9 100644
--- a/packages/api/src/generated/graphql.ts
+++ b/packages/api/src/generated/graphql.ts
@@ -1303,7 +1303,7 @@ export type HomeItem = {
   previewContent?: Maybe<Scalars['String']>;
   saveCount?: Maybe<Scalars['Int']>;
   seen_at?: Maybe<Scalars['Date']>;
-  subscription?: Maybe<HomeItemSource>;
+  source?: Maybe<HomeItemSource>;
   thumbnail?: Maybe<Scalars['String']>;
   title: Scalars['String'];
   url: Scalars['String'];
@@ -6043,7 +6043,7 @@ export type HomeItemResolvers<ContextType = ResolverContext, ParentType extends
   previewContent?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   saveCount?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
   seen_at?: Resolver<Maybe<ResolversTypes['Date']>, ParentType, ContextType>;
-  subscription?: Resolver<Maybe<ResolversTypes['HomeItemSource']>, ParentType, ContextType>;
+  source?: Resolver<Maybe<ResolversTypes['HomeItemSource']>, ParentType, ContextType>;
   thumbnail?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   title?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   url?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
diff --git a/packages/api/src/generated/schema.graphql b/packages/api/src/generated/schema.graphql
index 2b09a639f..220808a8a 100644
--- a/packages/api/src/generated/schema.graphql
+++ b/packages/api/src/generated/schema.graphql
@@ -1172,7 +1172,7 @@ type HomeItem {
   previewContent: String
   saveCount: Int
   seen_at: Date
-  subscription: HomeItemSource
+  source: HomeItemSource
   thumbnail: String
   title: String!
   url: String!
diff --git a/packages/api/src/resolvers/function_resolvers.ts b/packages/api/src/resolvers/function_resolvers.ts
index 213a774b7..6f4e84ee9 100644
--- a/packages/api/src/resolvers/function_resolvers.ts
+++ b/packages/api/src/resolvers/function_resolvers.ts
@@ -21,6 +21,8 @@ import {
   Article,
   Highlight,
   HomeItem,
+  HomeItemSource,
+  HomeItemSourceType,
   Label,
   PageType,
   Recommendation,
@@ -29,6 +31,7 @@ import {
 } from '../generated/graphql'
 import { getAISummary } from '../services/ai-summaries'
 import { findUserFeatures } from '../services/features'
+import { Merge } from '../util'
 import {
   highlightDataToHighlight,
   isBase64Image,
@@ -665,8 +668,10 @@ export const functionResolvers = {
               canDelete: !libraryItem.deletedAt,
               canSave: false,
               dir: libraryItem.directionality,
-              subscription: null,
               previewContent: libraryItem.description,
+              subscription: libraryItem.subscription,
+              siteName: libraryItem.siteName,
+              siteIcon: libraryItem.siteIcon,
             } as HomeItem)
         )
         .concat(
@@ -688,29 +693,44 @@ export const functionResolvers = {
                 broadcastCount: publicItem.stats.broadcastCount,
                 likeCount: publicItem.stats.likeCount,
                 saveCount: publicItem.stats.saveCount,
-                subscription: publicItem.source,
+                source: publicItem.source,
               } as HomeItem)
           )
         )
     },
   },
   HomeItem: {
-    async subscription(
-      item: HomeItem,
+    async source(
+      item: Merge<
+        HomeItem,
+        { subscription?: string; siteName: string; siteIcon?: string }
+      >,
       _: unknown,
       ctx: WithDataSourcesContext
-    ) {
-      // if (item.subscription) {
-      //   return item.subscription
-      // }
-      // const subscription = await ctx.dataLoaders.subscriptions.load(item.id)
-      // if (!subscription) {
-      //   return {
-      //     name: item.siteName,
-      //     icon: item.siteIcon,
-      //     type: 'rss',
-      //   }
-      // }
+    ): Promise<HomeItemSource> {
+      if (item.source) {
+        return item.source
+      }
+
+      if (!item.subscription) {
+        return {
+          name: item.siteName,
+          icon: item.siteIcon,
+          type: HomeItemSourceType.Library,
+        }
+      }
+
+      const subscription = await ctx.dataLoaders.subscriptions.load(
+        item.subscription
+      )
+
+      return {
+        id: subscription.id,
+        url: subscription.url,
+        name: subscription.name,
+        icon: subscription.icon,
+        type: subscription.type as unknown as HomeItemSourceType,
+      }
     },
   },
   ...resultResolveTypeResolver('Login'),
diff --git a/packages/api/src/resolvers/types.ts b/packages/api/src/resolvers/types.ts
index ef6adfeff..21b049bae 100644
--- a/packages/api/src/resolvers/types.ts
+++ b/packages/api/src/resolvers/types.ts
@@ -11,6 +11,7 @@ import { Label } from '../entity/label'
 import { LibraryItem } from '../entity/library_item'
 import { PublicItem } from '../entity/public_item'
 import { Recommendation } from '../entity/recommendation'
+import { Subscription } from '../entity/subscription'
 import { UploadFile } from '../entity/upload_file'
 import { HomeItem } from '../generated/graphql'
 import { PubsubClient } from '../pubsub'
@@ -56,6 +57,7 @@ export interface RequestContext {
     uploadFiles: DataLoader<string, UploadFile | undefined>
     libraryItems: DataLoader<string, LibraryItem>
     publicItems: DataLoader<string, PublicItem>
+    subscriptions: DataLoader<string, Subscription>
   }
 }
 
diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts
index ed74e5509..963adf81f 100755
--- a/packages/api/src/schema.ts
+++ b/packages/api/src/schema.ts
@@ -3130,7 +3130,7 @@ const schema = gql`
     dir: String
     seen_at: Date
     wordCount: Int
-    subscription: HomeItemSource
+    source: HomeItemSource
     canSave: Boolean
     canComment: Boolean
     canShare: Boolean

From a3d20c03951074c4a2d49b580c14c4f9c3934cf9 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 28 May 2024 15:05:44 +0800
Subject: [PATCH 32/33] add subscription to the feature

---
 packages/api/src/jobs/score_library_item.ts | 1 +
 packages/api/src/jobs/update_home.ts        | 1 +
 packages/api/src/services/score.ts          | 1 +
 3 files changed, 3 insertions(+)

diff --git a/packages/api/src/jobs/score_library_item.ts b/packages/api/src/jobs/score_library_item.ts
index 75f71f9dc..47279ce50 100644
--- a/packages/api/src/jobs/score_library_item.ts
+++ b/packages/api/src/jobs/score_library_item.ts
@@ -55,6 +55,7 @@ export const scoreLibraryItem = async (
       language: lanaugeToCode(libraryItem.itemLanguage || 'English'),
       word_count: libraryItem.wordCount,
       published_at: libraryItem.publishedAt,
+      subscription: libraryItem.subscription,
     } as Feature,
   }
 
diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts
index 26561795b..7a64d3661 100644
--- a/packages/api/src/jobs/update_home.ts
+++ b/packages/api/src/jobs/update_home.ts
@@ -182,6 +182,7 @@ const rankCandidates = async (
         author: item.author,
         word_count: item.wordCount,
         published_at: item.publishedAt,
+        subscription: item.subscription?.name,
       } as Feature
       return acc
     }, {} as Record<string, Feature>),
diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
index 9d41fead2..53bdbc3ce 100644
--- a/packages/api/src/services/score.ts
+++ b/packages/api/src/services/score.ts
@@ -12,6 +12,7 @@ export interface Feature {
   subscription_type?: string
   folder?: string
   published_at?: Date
+  subscription?: string
 }
 
 export interface ScoreApiRequestBody {

From 3d262a4d1f6703ebd2cc4da6d3ac7d245fb5d298 Mon Sep 17 00:00:00 2001
From: Hongbo Wu <hongbo@omnivore.app>
Date: Tue, 28 May 2024 15:44:53 +0800
Subject: [PATCH 33/33] remove fake code

---
 packages/api/src/jobs/update_home.ts     | 16 ++++++++++++----
 packages/api/src/resolvers/home/index.ts |  1 +
 packages/api/src/services/score.ts       |  7 -------
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/packages/api/src/jobs/update_home.ts b/packages/api/src/jobs/update_home.ts
index 7a64d3661..dd09639dc 100644
--- a/packages/api/src/jobs/update_home.ts
+++ b/packages/api/src/jobs/update_home.ts
@@ -15,6 +15,7 @@ export const UPDATE_HOME_JOB = 'UPDATE_HOME_JOB'
 
 export interface UpdateHomeJobData {
   userId: string
+  cursor?: number
 }
 
 interface Candidate {
@@ -249,7 +250,8 @@ export const getHomeSections = async (
 
 const appendSectionsToHome = async (
   userId: string,
-  sections: Array<Section>
+  sections: Array<Section>,
+  cursor = Date.now()
 ) => {
   const redisClient = redisDataSource.redisClient
   if (!redisClient) {
@@ -261,10 +263,14 @@ const appendSectionsToHome = async (
   // store candidates in redis sorted set
   const pipeline = redisClient.pipeline()
 
+  const offset = sections.length + 86_400_000
+  cursor = cursor - offset
+
   const scoreMembers = sections.flatMap((section, index) => [
-    Date.now() + index + 86_400_000, // sections expire in 24 hours
+    cursor + index + 86_400_000, // sections expire in 24 hours
     JSON.stringify(section),
   ])
+
   // add section to the sorted set
   pipeline.zadd(key, ...scoreMembers)
 
@@ -367,7 +373,9 @@ const mixHomeItems = (rankedHomeItems: Array<Candidate>): Array<Section> => {
 }
 
 export const updateHome = async (data: UpdateHomeJobData) => {
-  const { userId } = data
+  const { userId, cursor } = data
+  logger.info('Updating home for user', data)
+
   const user = await findActiveUser(userId)
   if (!user) {
     logger.error(`User ${userId} not found`)
@@ -395,6 +403,6 @@ export const updateHome = async (data: UpdateHomeJobData) => {
   logger.info(`Created ${rankedSections.length} sections`)
 
   logger.info('Appending sections to home')
-  await appendSectionsToHome(userId, rankedSections)
+  await appendSectionsToHome(userId, rankedSections, cursor)
   logger.info('Home updated for user', { userId })
 }
diff --git a/packages/api/src/resolvers/home/index.ts b/packages/api/src/resolvers/home/index.ts
index a153fd5c0..64a14503d 100644
--- a/packages/api/src/resolvers/home/index.ts
+++ b/packages/api/src/resolvers/home/index.ts
@@ -46,6 +46,7 @@ export const homeResolver = authorized<
 
     await enqueueUpdateHomeJob({
       userId: uid,
+      cursor,
     })
 
     log.info('Just read feed update enqueued')
diff --git a/packages/api/src/services/score.ts b/packages/api/src/services/score.ts
index 53bdbc3ce..2aaf1cf9b 100644
--- a/packages/api/src/services/score.ts
+++ b/packages/api/src/services/score.ts
@@ -47,11 +47,4 @@ export const getScores = async (
 
   const scores = (await response.json()) as ScoreApiResponse
   return scores
-
-  // // fake random scores
-  // const scores: ScoreApiResponse = {}
-  // for (const itemId of Object.keys(data.item_features)) {
-  //   scores[itemId] = Math.random()
-  // }
-  // return Promise.resolve(scores)
 }